diff options
Diffstat (limited to 'lib/Transforms/Instrumentation')
20 files changed, 2506 insertions, 2207 deletions
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp index f1558c75cb90b..6821e214e9216 100644 --- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -1,9 +1,8 @@ //===- AddressSanitizer.cpp - memory error detector -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,6 +12,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DepthFirstIterator.h" @@ -25,7 +25,6 @@ #include "llvm/ADT/Twine.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/IR/Argument.h" @@ -72,6 +71,7 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/ASanStackFrameLayout.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include <algorithm> @@ -94,9 +94,6 @@ static const uint64_t kDefaultShadowOffset32 = 1ULL << 29; static const uint64_t kDefaultShadowOffset64 = 1ULL << 44; static const uint64_t kDynamicShadowSentinel = std::numeric_limits<uint64_t>::max(); -static const uint64_t kIOSShadowOffset32 = 1ULL << 30; -static const uint64_t kIOSSimShadowOffset32 = 1ULL << 30; -static const uint64_t kIOSSimShadowOffset64 = kDefaultShadowOffset64; static const uint64_t kSmallX86_64ShadowOffsetBase = 0x7FFFFFFF; // < 2G. static const uint64_t kSmallX86_64ShadowOffsetAlignMask = ~0xFFFULL; static const uint64_t kLinuxKasan_ShadowOffset64 = 0xdffffc0000000000; @@ -112,6 +109,7 @@ static const uint64_t kNetBSD_ShadowOffset64 = 1ULL << 46; static const uint64_t kNetBSDKasan_ShadowOffset64 = 0xdfff900000000000; static const uint64_t kPS4CPU_ShadowOffset64 = 1ULL << 40; static const uint64_t kWindowsShadowOffset32 = 3ULL << 28; +static const uint64_t kEmscriptenShadowOffset = 0; static const uint64_t kMyriadShadowScale = 5; static const uint64_t kMyriadMemoryOffset32 = 0x80000000ULL; @@ -275,6 +273,16 @@ static cl::opt<bool> ClInvalidPointerPairs( cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden, cl::init(false)); +static cl::opt<bool> ClInvalidPointerCmp( + "asan-detect-invalid-pointer-cmp", + cl::desc("Instrument <, <=, >, >= with pointer operands"), cl::Hidden, + cl::init(false)); + +static cl::opt<bool> ClInvalidPointerSub( + "asan-detect-invalid-pointer-sub", + cl::desc("Instrument - operations with pointer operands"), cl::Hidden, + cl::init(false)); + static cl::opt<unsigned> ClRealignStack( "asan-realign-stack", cl::desc("Realign stack to the value of this flag (power of two)"), @@ -311,10 +319,10 @@ static cl::opt<int> ClMappingScale("asan-mapping-scale", cl::desc("scale of asan shadow mapping"), cl::Hidden, cl::init(0)); -static cl::opt<unsigned long long> ClMappingOffset( - "asan-mapping-offset", - cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"), cl::Hidden, - cl::init(0)); +static cl::opt<uint64_t> + ClMappingOffset("asan-mapping-offset", + cl::desc("offset of asan shadow mapping [EXPERIMENTAL]"), + cl::Hidden, cl::init(0)); // Optimization flags. Not user visible, used mostly for testing // and benchmarking the tool. @@ -393,87 +401,6 @@ STATISTIC(NumOptimizedAccessesToStackVar, namespace { -/// Frontend-provided metadata for source location. -struct LocationMetadata { - StringRef Filename; - int LineNo = 0; - int ColumnNo = 0; - - LocationMetadata() = default; - - bool empty() const { return Filename.empty(); } - - void parse(MDNode *MDN) { - assert(MDN->getNumOperands() == 3); - MDString *DIFilename = cast<MDString>(MDN->getOperand(0)); - Filename = DIFilename->getString(); - LineNo = - mdconst::extract<ConstantInt>(MDN->getOperand(1))->getLimitedValue(); - ColumnNo = - mdconst::extract<ConstantInt>(MDN->getOperand(2))->getLimitedValue(); - } -}; - -/// Frontend-provided metadata for global variables. -class GlobalsMetadata { -public: - struct Entry { - LocationMetadata SourceLoc; - StringRef Name; - bool IsDynInit = false; - bool IsBlacklisted = false; - - Entry() = default; - }; - - GlobalsMetadata() = default; - - void reset() { - inited_ = false; - Entries.clear(); - } - - void init(Module &M) { - assert(!inited_); - inited_ = true; - NamedMDNode *Globals = M.getNamedMetadata("llvm.asan.globals"); - if (!Globals) return; - for (auto MDN : Globals->operands()) { - // Metadata node contains the global and the fields of "Entry". - assert(MDN->getNumOperands() == 5); - auto *V = mdconst::extract_or_null<Constant>(MDN->getOperand(0)); - // The optimizer may optimize away a global entirely. - if (!V) continue; - auto *StrippedV = V->stripPointerCasts(); - auto *GV = dyn_cast<GlobalVariable>(StrippedV); - if (!GV) continue; - // We can already have an entry for GV if it was merged with another - // global. - Entry &E = Entries[GV]; - if (auto *Loc = cast_or_null<MDNode>(MDN->getOperand(1))) - E.SourceLoc.parse(Loc); - if (auto *Name = cast_or_null<MDString>(MDN->getOperand(2))) - E.Name = Name->getString(); - ConstantInt *IsDynInit = - mdconst::extract<ConstantInt>(MDN->getOperand(3)); - E.IsDynInit |= IsDynInit->isOne(); - ConstantInt *IsBlacklisted = - mdconst::extract<ConstantInt>(MDN->getOperand(4)); - E.IsBlacklisted |= IsBlacklisted->isOne(); - } - } - - /// Returns metadata entry for a given global. - Entry get(GlobalVariable *G) const { - auto Pos = Entries.find(G); - return (Pos != Entries.end()) ? Pos->second : Entry(); - } - -private: - bool inited_ = false; - DenseMap<GlobalVariable *, Entry> Entries; -}; - /// This struct defines the shadow mapping using the rule: /// shadow = (mem >> Scale) ADD-or-OR Offset. /// If InGlobal is true, then @@ -499,7 +426,6 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, bool IsPPC64 = TargetTriple.getArch() == Triple::ppc64 || TargetTriple.getArch() == Triple::ppc64le; bool IsSystemZ = TargetTriple.getArch() == Triple::systemz; - bool IsX86 = TargetTriple.getArch() == Triple::x86; bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64; bool IsMIPS32 = TargetTriple.isMIPS32(); bool IsMIPS64 = TargetTriple.isMIPS64(); @@ -508,6 +434,7 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, bool IsWindows = TargetTriple.isOSWindows(); bool IsFuchsia = TargetTriple.isOSFuchsia(); bool IsMyriad = TargetTriple.getVendor() == llvm::Triple::Myriad; + bool IsEmscripten = TargetTriple.isOSEmscripten(); ShadowMapping Mapping; @@ -526,10 +453,11 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, else if (IsNetBSD) Mapping.Offset = kNetBSD_ShadowOffset32; else if (IsIOS) - // If we're targeting iOS and x86, the binary is built for iOS simulator. - Mapping.Offset = IsX86 ? kIOSSimShadowOffset32 : kIOSShadowOffset32; + Mapping.Offset = kDynamicShadowSentinel; else if (IsWindows) Mapping.Offset = kWindowsShadowOffset32; + else if (IsEmscripten) + Mapping.Offset = kEmscriptenShadowOffset; else if (IsMyriad) { uint64_t ShadowOffset = (kMyriadMemoryOffset32 + kMyriadMemorySize32 - (kMyriadMemorySize32 >> Mapping.Scale)); @@ -566,10 +494,7 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize, } else if (IsMIPS64) Mapping.Offset = kMIPS64_ShadowOffset64; else if (IsIOS) - // If we're targeting iOS and x86, the binary is built for iOS simulator. - // We are using dynamic shadow offset on the 64-bit devices. - Mapping.Offset = - IsX86_64 ? kIOSSimShadowOffset64 : kDynamicShadowSentinel; + Mapping.Offset = kDynamicShadowSentinel; else if (IsAArch64) Mapping.Offset = kAArch64_ShadowOffset64; else @@ -607,27 +532,53 @@ static size_t RedzoneSizeForScale(int MappingScale) { namespace { -/// AddressSanitizer: instrument the code in module to find memory bugs. -struct AddressSanitizer : public FunctionPass { - // Pass identification, replacement for typeid +/// Module analysis for getting various metadata about the module. +class ASanGlobalsMetadataWrapperPass : public ModulePass { +public: static char ID; - explicit AddressSanitizer(bool CompileKernel = false, bool Recover = false, - bool UseAfterScope = false) - : FunctionPass(ID), UseAfterScope(UseAfterScope || ClUseAfterScope) { - this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; - this->CompileKernel = ClEnableKasan.getNumOccurrences() > 0 ? - ClEnableKasan : CompileKernel; - initializeAddressSanitizerPass(*PassRegistry::getPassRegistry()); + ASanGlobalsMetadataWrapperPass() : ModulePass(ID) { + initializeASanGlobalsMetadataWrapperPassPass( + *PassRegistry::getPassRegistry()); + } + + bool runOnModule(Module &M) override { + GlobalsMD = GlobalsMetadata(M); + return false; } StringRef getPassName() const override { - return "AddressSanitizerFunctionPass"; + return "ASanGlobalsMetadataWrapperPass"; } void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired<DominatorTreeWrapperPass>(); - AU.addRequired<TargetLibraryInfoWrapperPass>(); + AU.setPreservesAll(); + } + + GlobalsMetadata &getGlobalsMD() { return GlobalsMD; } + +private: + GlobalsMetadata GlobalsMD; +}; + +char ASanGlobalsMetadataWrapperPass::ID = 0; + +/// AddressSanitizer: instrument the code in module to find memory bugs. +struct AddressSanitizer { + AddressSanitizer(Module &M, GlobalsMetadata &GlobalsMD, + bool CompileKernel = false, bool Recover = false, + bool UseAfterScope = false) + : UseAfterScope(UseAfterScope || ClUseAfterScope), GlobalsMD(GlobalsMD) { + this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; + this->CompileKernel = + ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan : CompileKernel; + + C = &(M.getContext()); + LongSize = M.getDataLayout().getPointerSizeInBits(); + IntptrTy = Type::getIntNTy(*C, LongSize); + TargetTriple = Triple(M.getTargetTriple()); + + Mapping = getShadowMapping(TargetTriple, LongSize, this->CompileKernel); } uint64_t getAllocaSizeInBytes(const AllocaInst &AI) const { @@ -672,14 +623,10 @@ struct AddressSanitizer : public FunctionPass { Value *SizeArgument, uint32_t Exp); void instrumentMemIntrinsic(MemIntrinsic *MI); Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); - bool runOnFunction(Function &F) override; + bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI); bool maybeInsertAsanInitAtFunctionEntry(Function &F); void maybeInsertDynamicShadowAtFunctionEntry(Function &F); void markEscapedLocalAllocas(Function &F); - bool doInitialization(Module &M) override; - bool doFinalization(Module &M) override; - - DominatorTree &getDominatorTree() const { return *DT; } private: friend struct FunctionStackPoisoner; @@ -715,36 +662,68 @@ private: bool UseAfterScope; Type *IntptrTy; ShadowMapping Mapping; - DominatorTree *DT; - Function *AsanHandleNoReturnFunc; - Function *AsanPtrCmpFunction, *AsanPtrSubFunction; + FunctionCallee AsanHandleNoReturnFunc; + FunctionCallee AsanPtrCmpFunction, AsanPtrSubFunction; Constant *AsanShadowGlobal; // These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize). - Function *AsanErrorCallback[2][2][kNumberOfAccessSizes]; - Function *AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes]; + FunctionCallee AsanErrorCallback[2][2][kNumberOfAccessSizes]; + FunctionCallee AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes]; // These arrays is indexed by AccessIsWrite and Experiment. - Function *AsanErrorCallbackSized[2][2]; - Function *AsanMemoryAccessCallbackSized[2][2]; + FunctionCallee AsanErrorCallbackSized[2][2]; + FunctionCallee AsanMemoryAccessCallbackSized[2][2]; - Function *AsanMemmove, *AsanMemcpy, *AsanMemset; + FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset; InlineAsm *EmptyAsm; Value *LocalDynamicShadow = nullptr; GlobalsMetadata GlobalsMD; DenseMap<const AllocaInst *, bool> ProcessedAllocas; }; -class AddressSanitizerModule : public ModulePass { +class AddressSanitizerLegacyPass : public FunctionPass { public: - // Pass identification, replacement for typeid static char ID; - explicit AddressSanitizerModule(bool CompileKernel = false, - bool Recover = false, - bool UseGlobalsGC = true, - bool UseOdrIndicator = false) - : ModulePass(ID), UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC), + explicit AddressSanitizerLegacyPass(bool CompileKernel = false, + bool Recover = false, + bool UseAfterScope = false) + : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover), + UseAfterScope(UseAfterScope) { + initializeAddressSanitizerLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "AddressSanitizerFunctionPass"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<ASanGlobalsMetadataWrapperPass>(); + AU.addRequired<TargetLibraryInfoWrapperPass>(); + } + + bool runOnFunction(Function &F) override { + GlobalsMetadata &GlobalsMD = + getAnalysis<ASanGlobalsMetadataWrapperPass>().getGlobalsMD(); + const TargetLibraryInfo *TLI = + &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); + AddressSanitizer ASan(*F.getParent(), GlobalsMD, CompileKernel, Recover, + UseAfterScope); + return ASan.instrumentFunction(F, TLI); + } + +private: + bool CompileKernel; + bool Recover; + bool UseAfterScope; +}; + +class ModuleAddressSanitizer { +public: + ModuleAddressSanitizer(Module &M, GlobalsMetadata &GlobalsMD, + bool CompileKernel = false, bool Recover = false, + bool UseGlobalsGC = true, bool UseOdrIndicator = false) + : GlobalsMD(GlobalsMD), UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC), // Enable aliases as they should have no downside with ODR indicators. UsePrivateAlias(UseOdrIndicator || ClUsePrivateAlias), UseOdrIndicator(UseOdrIndicator || ClUseOdrIndicator), @@ -759,10 +738,15 @@ public: this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; this->CompileKernel = ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan : CompileKernel; + + C = &(M.getContext()); + int LongSize = M.getDataLayout().getPointerSizeInBits(); + IntptrTy = Type::getIntNTy(*C, LongSize); + TargetTriple = Triple(M.getTargetTriple()); + Mapping = getShadowMapping(TargetTriple, LongSize, this->CompileKernel); } - bool runOnModule(Module &M) override; - StringRef getPassName() const override { return "AddressSanitizerModule"; } + bool instrumentModule(Module &); private: void initializeCallbacks(Module &M); @@ -810,19 +794,54 @@ private: LLVMContext *C; Triple TargetTriple; ShadowMapping Mapping; - Function *AsanPoisonGlobals; - Function *AsanUnpoisonGlobals; - Function *AsanRegisterGlobals; - Function *AsanUnregisterGlobals; - Function *AsanRegisterImageGlobals; - Function *AsanUnregisterImageGlobals; - Function *AsanRegisterElfGlobals; - Function *AsanUnregisterElfGlobals; + FunctionCallee AsanPoisonGlobals; + FunctionCallee AsanUnpoisonGlobals; + FunctionCallee AsanRegisterGlobals; + FunctionCallee AsanUnregisterGlobals; + FunctionCallee AsanRegisterImageGlobals; + FunctionCallee AsanUnregisterImageGlobals; + FunctionCallee AsanRegisterElfGlobals; + FunctionCallee AsanUnregisterElfGlobals; Function *AsanCtorFunction = nullptr; Function *AsanDtorFunction = nullptr; }; +class ModuleAddressSanitizerLegacyPass : public ModulePass { +public: + static char ID; + + explicit ModuleAddressSanitizerLegacyPass(bool CompileKernel = false, + bool Recover = false, + bool UseGlobalGC = true, + bool UseOdrIndicator = false) + : ModulePass(ID), CompileKernel(CompileKernel), Recover(Recover), + UseGlobalGC(UseGlobalGC), UseOdrIndicator(UseOdrIndicator) { + initializeModuleAddressSanitizerLegacyPassPass( + *PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "ModuleAddressSanitizer"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<ASanGlobalsMetadataWrapperPass>(); + } + + bool runOnModule(Module &M) override { + GlobalsMetadata &GlobalsMD = + getAnalysis<ASanGlobalsMetadataWrapperPass>().getGlobalsMD(); + ModuleAddressSanitizer ASanModule(M, GlobalsMD, CompileKernel, Recover, + UseGlobalGC, UseOdrIndicator); + return ASanModule.instrumentModule(M); + } + +private: + bool CompileKernel; + bool Recover; + bool UseGlobalGC; + bool UseOdrIndicator; +}; + // Stack poisoning does not play well with exception handling. // When an exception is thrown, we essentially bypass the code // that unpoisones the stack. This is why the run-time library has @@ -846,11 +865,11 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { SmallVector<Instruction *, 8> RetVec; unsigned StackAlignment; - Function *AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1], - *AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1]; - Function *AsanSetShadowFunc[0x100] = {}; - Function *AsanPoisonStackMemoryFunc, *AsanUnpoisonStackMemoryFunc; - Function *AsanAllocaPoisonFunc, *AsanAllocasUnpoisonFunc; + FunctionCallee AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1], + AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1]; + FunctionCallee AsanSetShadowFunc[0x100] = {}; + FunctionCallee AsanPoisonStackMemoryFunc, AsanUnpoisonStackMemoryFunc; + FunctionCallee AsanAllocaPoisonFunc, AsanAllocasUnpoisonFunc; // Stores a place and arguments of poisoning/unpoisoning call for alloca. struct AllocaPoisonCall { @@ -861,6 +880,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { }; SmallVector<AllocaPoisonCall, 8> DynamicAllocaPoisonCallVec; SmallVector<AllocaPoisonCall, 8> StaticAllocaPoisonCallVec; + bool HasUntracedLifetimeIntrinsic = false; SmallVector<AllocaInst *, 1> DynamicAllocaVec; SmallVector<IntrinsicInst *, 1> StackRestoreVec; @@ -876,13 +896,9 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { std::unique_ptr<CallInst> EmptyInlineAsm; FunctionStackPoisoner(Function &F, AddressSanitizer &ASan) - : F(F), - ASan(ASan), - DIB(*F.getParent(), /*AllowUnresolved*/ false), - C(ASan.C), - IntptrTy(ASan.IntptrTy), - IntptrPtrTy(PointerType::get(IntptrTy, 0)), - Mapping(ASan.Mapping), + : F(F), ASan(ASan), DIB(*F.getParent(), /*AllowUnresolved*/ false), + C(ASan.C), IntptrTy(ASan.IntptrTy), + IntptrPtrTy(PointerType::get(IntptrTy, 0)), Mapping(ASan.Mapping), StackAlignment(1 << Mapping.Scale), EmptyInlineAsm(CallInst::Create(ASan.EmptyAsm)) {} @@ -899,6 +915,14 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { initializeCallbacks(*F.getParent()); + if (HasUntracedLifetimeIntrinsic) { + // If there are lifetime intrinsics which couldn't be traced back to an + // alloca, we may not know exactly when a variable enters scope, and + // therefore should "fail safe" by not poisoning them. + StaticAllocaPoisonCallVec.clear(); + DynamicAllocaPoisonCallVec.clear(); + } + processDynamicAllocas(); processStaticAllocas(); @@ -950,8 +974,9 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { DynamicAreaOffset); } - IRB.CreateCall(AsanAllocasUnpoisonFunc, - {IRB.CreateLoad(DynamicAllocaLayout), DynamicAreaPtr}); + IRB.CreateCall( + AsanAllocasUnpoisonFunc, + {IRB.CreateLoad(IntptrTy, DynamicAllocaLayout), DynamicAreaPtr}); } // Unpoison dynamic allocas redzones. @@ -1018,8 +1043,14 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { !ConstantInt::isValueValidForType(IntptrTy, SizeValue)) return; // Find alloca instruction that corresponds to llvm.lifetime argument. - AllocaInst *AI = findAllocaForValue(II.getArgOperand(1)); - if (!AI || !ASan.isInterestingAlloca(*AI)) + AllocaInst *AI = + llvm::findAllocaForValue(II.getArgOperand(1), AllocaForValue); + if (!AI) { + HasUntracedLifetimeIntrinsic = true; + return; + } + // We're interested only in allocas we can handle. + if (!ASan.isInterestingAlloca(*AI)) return; bool DoPoison = (ID == Intrinsic::lifetime_end); AllocaPoisonCall APC = {&II, AI, SizeValue, DoPoison}; @@ -1042,16 +1073,6 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { // ---------------------- Helpers. void initializeCallbacks(Module &M); - bool doesDominateAllExits(const Instruction *I) const { - for (auto Ret : RetVec) { - if (!ASan.getDominatorTree().dominates(I, Ret)) return false; - } - return true; - } - - /// Finds alloca where the value comes from. - AllocaInst *findAllocaForValue(Value *V); - // Copies bytes from ShadowBytes into shadow memory for indexes where // ShadowMask is not zero. If ShadowMask[i] is zero, we assume that // ShadowBytes[i] is constantly zero and doesn't need to be overwritten. @@ -1074,16 +1095,111 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> { } // end anonymous namespace -char AddressSanitizer::ID = 0; +void LocationMetadata::parse(MDNode *MDN) { + assert(MDN->getNumOperands() == 3); + MDString *DIFilename = cast<MDString>(MDN->getOperand(0)); + Filename = DIFilename->getString(); + LineNo = mdconst::extract<ConstantInt>(MDN->getOperand(1))->getLimitedValue(); + ColumnNo = + mdconst::extract<ConstantInt>(MDN->getOperand(2))->getLimitedValue(); +} + +// FIXME: It would be cleaner to instead attach relevant metadata to the globals +// we want to sanitize instead and reading this metadata on each pass over a +// function instead of reading module level metadata at first. +GlobalsMetadata::GlobalsMetadata(Module &M) { + NamedMDNode *Globals = M.getNamedMetadata("llvm.asan.globals"); + if (!Globals) + return; + for (auto MDN : Globals->operands()) { + // Metadata node contains the global and the fields of "Entry". + assert(MDN->getNumOperands() == 5); + auto *V = mdconst::extract_or_null<Constant>(MDN->getOperand(0)); + // The optimizer may optimize away a global entirely. + if (!V) + continue; + auto *StrippedV = V->stripPointerCasts(); + auto *GV = dyn_cast<GlobalVariable>(StrippedV); + if (!GV) + continue; + // We can already have an entry for GV if it was merged with another + // global. + Entry &E = Entries[GV]; + if (auto *Loc = cast_or_null<MDNode>(MDN->getOperand(1))) + E.SourceLoc.parse(Loc); + if (auto *Name = cast_or_null<MDString>(MDN->getOperand(2))) + E.Name = Name->getString(); + ConstantInt *IsDynInit = mdconst::extract<ConstantInt>(MDN->getOperand(3)); + E.IsDynInit |= IsDynInit->isOne(); + ConstantInt *IsBlacklisted = + mdconst::extract<ConstantInt>(MDN->getOperand(4)); + E.IsBlacklisted |= IsBlacklisted->isOne(); + } +} + +AnalysisKey ASanGlobalsMetadataAnalysis::Key; + +GlobalsMetadata ASanGlobalsMetadataAnalysis::run(Module &M, + ModuleAnalysisManager &AM) { + return GlobalsMetadata(M); +} + +AddressSanitizerPass::AddressSanitizerPass(bool CompileKernel, bool Recover, + bool UseAfterScope) + : CompileKernel(CompileKernel), Recover(Recover), + UseAfterScope(UseAfterScope) {} + +PreservedAnalyses AddressSanitizerPass::run(Function &F, + AnalysisManager<Function> &AM) { + auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F); + auto &MAM = MAMProxy.getManager(); + Module &M = *F.getParent(); + if (auto *R = MAM.getCachedResult<ASanGlobalsMetadataAnalysis>(M)) { + const TargetLibraryInfo *TLI = &AM.getResult<TargetLibraryAnalysis>(F); + AddressSanitizer Sanitizer(M, *R, CompileKernel, Recover, UseAfterScope); + if (Sanitizer.instrumentFunction(F, TLI)) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); + } + + report_fatal_error( + "The ASanGlobalsMetadataAnalysis is required to run before " + "AddressSanitizer can run"); + return PreservedAnalyses::all(); +} + +ModuleAddressSanitizerPass::ModuleAddressSanitizerPass(bool CompileKernel, + bool Recover, + bool UseGlobalGC, + bool UseOdrIndicator) + : CompileKernel(CompileKernel), Recover(Recover), UseGlobalGC(UseGlobalGC), + UseOdrIndicator(UseOdrIndicator) {} + +PreservedAnalyses ModuleAddressSanitizerPass::run(Module &M, + AnalysisManager<Module> &AM) { + GlobalsMetadata &GlobalsMD = AM.getResult<ASanGlobalsMetadataAnalysis>(M); + ModuleAddressSanitizer Sanitizer(M, GlobalsMD, CompileKernel, Recover, + UseGlobalGC, UseOdrIndicator); + if (Sanitizer.instrumentModule(M)) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +INITIALIZE_PASS(ASanGlobalsMetadataWrapperPass, "asan-globals-md", + "Read metadata to mark which globals should be instrumented " + "when running ASan.", + false, true) + +char AddressSanitizerLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN( - AddressSanitizer, "asan", + AddressSanitizerLegacyPass, "asan", "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false, false) -INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(ASanGlobalsMetadataWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_END( - AddressSanitizer, "asan", + AddressSanitizerLegacyPass, "asan", "AddressSanitizer: detects use-after-free and out-of-bounds bugs.", false, false) @@ -1091,24 +1207,22 @@ FunctionPass *llvm::createAddressSanitizerFunctionPass(bool CompileKernel, bool Recover, bool UseAfterScope) { assert(!CompileKernel || Recover); - return new AddressSanitizer(CompileKernel, Recover, UseAfterScope); + return new AddressSanitizerLegacyPass(CompileKernel, Recover, UseAfterScope); } -char AddressSanitizerModule::ID = 0; +char ModuleAddressSanitizerLegacyPass::ID = 0; INITIALIZE_PASS( - AddressSanitizerModule, "asan-module", + ModuleAddressSanitizerLegacyPass, "asan-module", "AddressSanitizer: detects use-after-free and out-of-bounds bugs." "ModulePass", false, false) -ModulePass *llvm::createAddressSanitizerModulePass(bool CompileKernel, - bool Recover, - bool UseGlobalsGC, - bool UseOdrIndicator) { +ModulePass *llvm::createModuleAddressSanitizerLegacyPassPass( + bool CompileKernel, bool Recover, bool UseGlobalsGC, bool UseOdrIndicator) { assert(!CompileKernel || Recover); - return new AddressSanitizerModule(CompileKernel, Recover, UseGlobalsGC, - UseOdrIndicator); + return new ModuleAddressSanitizerLegacyPass(CompileKernel, Recover, + UseGlobalsGC, UseOdrIndicator); } static size_t TypeSizeToSizeIndex(uint32_t TypeSize) { @@ -1312,11 +1426,24 @@ static bool isPointerOperand(Value *V) { // This is a rough heuristic; it may cause both false positives and // false negatives. The proper implementation requires cooperation with // the frontend. -static bool isInterestingPointerComparisonOrSubtraction(Instruction *I) { +static bool isInterestingPointerComparison(Instruction *I) { if (ICmpInst *Cmp = dyn_cast<ICmpInst>(I)) { - if (!Cmp->isRelational()) return false; - } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) { - if (BO->getOpcode() != Instruction::Sub) return false; + if (!Cmp->isRelational()) + return false; + } else { + return false; + } + return isPointerOperand(I->getOperand(0)) && + isPointerOperand(I->getOperand(1)); +} + +// This is a rough heuristic; it may cause both false positives and +// false negatives. The proper implementation requires cooperation with +// the frontend. +static bool isInterestingPointerSubtraction(Instruction *I) { + if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) { + if (BO->getOpcode() != Instruction::Sub) + return false; } else { return false; } @@ -1328,13 +1455,16 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) { // If a global variable does not have dynamic initialization we don't // have to instrument it. However, if a global does not have initializer // at all, we assume it has dynamic initializer (in other TU). + // + // FIXME: Metadata should be attched directly to the global directly instead + // of being added to llvm.asan.globals. return G->hasInitializer() && !GlobalsMD.get(G).IsDynInit; } void AddressSanitizer::instrumentPointerComparisonOrSubtraction( Instruction *I) { IRBuilder<> IRB(I); - Function *F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction; + FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction; Value *Param[2] = {I->getOperand(0), I->getOperand(1)}; for (Value *&i : Param) { if (i->getType()->isPointerTy()) @@ -1392,7 +1522,7 @@ static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, IRBuilder<> IRB(InsertBefore); InstrumentedAddress = - IRB.CreateGEP(Addr, {Zero, ConstantInt::get(IntptrTy, Idx)}); + IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)}); doInstrumentAddress(Pass, I, InsertBefore, InstrumentedAddress, Alignment, Granularity, ElemTypeSize, IsWrite, SizeArgument, UseCalls, Exp); @@ -1553,7 +1683,7 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, Value *ShadowPtr = memToShadow(AddrLong, IRB); Value *CmpVal = Constant::getNullValue(ShadowTy); Value *ShadowValue = - IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); + IRB.CreateLoad(ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); Value *Cmp = IRB.CreateICmpNE(ShadowValue, CmpVal); size_t Granularity = 1ULL << Mapping.Scale; @@ -1612,7 +1742,7 @@ void AddressSanitizer::instrumentUnusualSizeOrAlignment( } } -void AddressSanitizerModule::poisonOneInitializer(Function &GlobalInit, +void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName) { // Set up the arguments to our poison/unpoison functions. IRBuilder<> IRB(&GlobalInit.front(), @@ -1628,7 +1758,7 @@ void AddressSanitizerModule::poisonOneInitializer(Function &GlobalInit, CallInst::Create(AsanUnpoisonGlobals, "", RI); } -void AddressSanitizerModule::createInitializerPoisonCalls( +void ModuleAddressSanitizer::createInitializerPoisonCalls( Module &M, GlobalValue *ModuleName) { GlobalVariable *GV = M.getGlobalVariable("llvm.global_ctors"); if (!GV) @@ -1653,10 +1783,12 @@ void AddressSanitizerModule::createInitializerPoisonCalls( } } -bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) { +bool ModuleAddressSanitizer::ShouldInstrumentGlobal(GlobalVariable *G) { Type *Ty = G->getValueType(); LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n"); + // FIXME: Metadata should be attched directly to the global directly instead + // of being added to llvm.asan.globals. if (GlobalsMD.get(G).IsBlacklisted) return false; if (!Ty->isSized()) return false; if (!G->hasInitializer()) return false; @@ -1768,7 +1900,7 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) { // On Mach-O platforms, we emit global metadata in a separate section of the // binary in order to allow the linker to properly dead strip. This is only // supported on recent versions of ld64. -bool AddressSanitizerModule::ShouldUseMachOGlobalsSection() const { +bool ModuleAddressSanitizer::ShouldUseMachOGlobalsSection() const { if (!TargetTriple.isOSBinFormatMachO()) return false; @@ -1782,7 +1914,7 @@ bool AddressSanitizerModule::ShouldUseMachOGlobalsSection() const { return false; } -StringRef AddressSanitizerModule::getGlobalMetadataSection() const { +StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const { switch (TargetTriple.getObjectFormat()) { case Triple::COFF: return ".ASAN$GL"; case Triple::ELF: return "asan_globals"; @@ -1792,52 +1924,39 @@ StringRef AddressSanitizerModule::getGlobalMetadataSection() const { llvm_unreachable("unsupported object format"); } -void AddressSanitizerModule::initializeCallbacks(Module &M) { +void ModuleAddressSanitizer::initializeCallbacks(Module &M) { IRBuilder<> IRB(*C); // Declare our poisoning and unpoisoning functions. - AsanPoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy)); - AsanPoisonGlobals->setLinkage(Function::ExternalLinkage); - AsanUnpoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanUnpoisonGlobalsName, IRB.getVoidTy())); - AsanUnpoisonGlobals->setLinkage(Function::ExternalLinkage); + AsanPoisonGlobals = + M.getOrInsertFunction(kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy); + AsanUnpoisonGlobals = + M.getOrInsertFunction(kAsanUnpoisonGlobalsName, IRB.getVoidTy()); // Declare functions that register/unregister globals. - AsanRegisterGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy)); - AsanRegisterGlobals->setLinkage(Function::ExternalLinkage); - AsanUnregisterGlobals = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(kAsanUnregisterGlobalsName, IRB.getVoidTy(), - IntptrTy, IntptrTy)); - AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage); + AsanRegisterGlobals = M.getOrInsertFunction( + kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy); + AsanUnregisterGlobals = M.getOrInsertFunction( + kAsanUnregisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy); // Declare the functions that find globals in a shared object and then invoke // the (un)register function on them. - AsanRegisterImageGlobals = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanRegisterImageGlobalsName, IRB.getVoidTy(), IntptrTy)); - AsanRegisterImageGlobals->setLinkage(Function::ExternalLinkage); + AsanRegisterImageGlobals = M.getOrInsertFunction( + kAsanRegisterImageGlobalsName, IRB.getVoidTy(), IntptrTy); + AsanUnregisterImageGlobals = M.getOrInsertFunction( + kAsanUnregisterImageGlobalsName, IRB.getVoidTy(), IntptrTy); - AsanUnregisterImageGlobals = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanUnregisterImageGlobalsName, IRB.getVoidTy(), IntptrTy)); - AsanUnregisterImageGlobals->setLinkage(Function::ExternalLinkage); - - AsanRegisterElfGlobals = checkSanitizerInterfaceFunction( + AsanRegisterElfGlobals = M.getOrInsertFunction(kAsanRegisterElfGlobalsName, IRB.getVoidTy(), - IntptrTy, IntptrTy, IntptrTy)); - AsanRegisterElfGlobals->setLinkage(Function::ExternalLinkage); - - AsanUnregisterElfGlobals = checkSanitizerInterfaceFunction( + IntptrTy, IntptrTy, IntptrTy); + AsanUnregisterElfGlobals = M.getOrInsertFunction(kAsanUnregisterElfGlobalsName, IRB.getVoidTy(), - IntptrTy, IntptrTy, IntptrTy)); - AsanUnregisterElfGlobals->setLinkage(Function::ExternalLinkage); + IntptrTy, IntptrTy, IntptrTy); } // Put the metadata and the instrumented global in the same group. This ensures // that the metadata is discarded if the instrumented global is discarded. -void AddressSanitizerModule::SetComdatForGlobalMetadata( +void ModuleAddressSanitizer::SetComdatForGlobalMetadata( GlobalVariable *G, GlobalVariable *Metadata, StringRef InternalSuffix) { Module &M = *G->getParent(); Comdat *C = G->getComdat(); @@ -1875,7 +1994,7 @@ void AddressSanitizerModule::SetComdatForGlobalMetadata( // Create a separate metadata global and put it in the appropriate ASan // global registration section. GlobalVariable * -AddressSanitizerModule::CreateMetadataGlobal(Module &M, Constant *Initializer, +ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer, StringRef OriginalName) { auto Linkage = TargetTriple.isOSBinFormatMachO() ? GlobalVariable::InternalLinkage @@ -1887,7 +2006,7 @@ AddressSanitizerModule::CreateMetadataGlobal(Module &M, Constant *Initializer, return Metadata; } -IRBuilder<> AddressSanitizerModule::CreateAsanModuleDtor(Module &M) { +IRBuilder<> ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) { AsanDtorFunction = Function::Create(FunctionType::get(Type::getVoidTy(*C), false), GlobalValue::InternalLinkage, kAsanModuleDtorName, &M); @@ -1896,7 +2015,7 @@ IRBuilder<> AddressSanitizerModule::CreateAsanModuleDtor(Module &M) { return IRBuilder<>(ReturnInst::Create(*C, AsanDtorBB)); } -void AddressSanitizerModule::InstrumentGlobalsCOFF( +void ModuleAddressSanitizer::InstrumentGlobalsCOFF( IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals, ArrayRef<Constant *> MetadataInitializers) { assert(ExtendedGlobals.size() == MetadataInitializers.size()); @@ -1920,7 +2039,7 @@ void AddressSanitizerModule::InstrumentGlobalsCOFF( } } -void AddressSanitizerModule::InstrumentGlobalsELF( +void ModuleAddressSanitizer::InstrumentGlobalsELF( IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals, ArrayRef<Constant *> MetadataInitializers, const std::string &UniqueModuleId) { @@ -1979,7 +2098,7 @@ void AddressSanitizerModule::InstrumentGlobalsELF( IRB.CreatePointerCast(StopELFMetadata, IntptrTy)}); } -void AddressSanitizerModule::InstrumentGlobalsMachO( +void ModuleAddressSanitizer::InstrumentGlobalsMachO( IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals, ArrayRef<Constant *> MetadataInitializers) { assert(ExtendedGlobals.size() == MetadataInitializers.size()); @@ -2036,7 +2155,7 @@ void AddressSanitizerModule::InstrumentGlobalsMachO( {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)}); } -void AddressSanitizerModule::InstrumentGlobalsWithMetadataArray( +void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray( IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals, ArrayRef<Constant *> MetadataInitializers) { assert(ExtendedGlobals.size() == MetadataInitializers.size()); @@ -2070,9 +2189,9 @@ void AddressSanitizerModule::InstrumentGlobalsWithMetadataArray( // redzones and inserts this function into llvm.global_ctors. // Sets *CtorComdat to true if the global registration code emitted into the // asan constructor is comdat-compatible. -bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat) { +bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M, + bool *CtorComdat) { *CtorComdat = false; - GlobalsMD.init(M); SmallVector<GlobalVariable *, 16> GlobalsToChange; @@ -2115,6 +2234,8 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool static const uint64_t kMaxGlobalRedzone = 1 << 18; GlobalVariable *G = GlobalsToChange[i]; + // FIXME: Metadata should be attched directly to the global directly instead + // of being added to llvm.asan.globals. auto MD = GlobalsMD.get(G); StringRef NameForGlobal = G->getName(); // Create string holding the global name (use global name from metadata @@ -2271,7 +2392,7 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool return true; } -int AddressSanitizerModule::GetAsanVersion(const Module &M) const { +int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const { int LongSize = M.getDataLayout().getPointerSizeInBits(); bool isAndroid = Triple(M.getTargetTriple()).isAndroid(); int Version = 8; @@ -2281,12 +2402,7 @@ int AddressSanitizerModule::GetAsanVersion(const Module &M) const { return Version; } -bool AddressSanitizerModule::runOnModule(Module &M) { - C = &(M.getContext()); - int LongSize = M.getDataLayout().getPointerSizeInBits(); - IntptrTy = Type::getIntNTy(*C, LongSize); - TargetTriple = Triple(M.getTargetTriple()); - Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); +bool ModuleAddressSanitizer::instrumentModule(Module &M) { initializeCallbacks(M); if (CompileKernel) @@ -2346,51 +2462,49 @@ void AddressSanitizer::initializeCallbacks(Module &M) { Args2.push_back(ExpType); Args1.push_back(ExpType); } - AsanErrorCallbackSized[AccessIsWrite][Exp] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr, - FunctionType::get(IRB.getVoidTy(), Args2, false))); + AsanErrorCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction( + kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr, + FunctionType::get(IRB.getVoidTy(), Args2, false)); - AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr, - FunctionType::get(IRB.getVoidTy(), Args2, false))); + AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction( + ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr, + FunctionType::get(IRB.getVoidTy(), Args2, false)); for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; AccessSizeIndex++) { const std::string Suffix = TypeStr + itostr(1ULL << AccessSizeIndex); AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( + M.getOrInsertFunction( kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr, - FunctionType::get(IRB.getVoidTy(), Args1, false))); + FunctionType::get(IRB.getVoidTy(), Args1, false)); AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( + M.getOrInsertFunction( ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr, - FunctionType::get(IRB.getVoidTy(), Args1, false))); + FunctionType::get(IRB.getVoidTy(), Args1, false)); } } } const std::string MemIntrinCallbackPrefix = CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix; - AsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); - AsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); - AsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy)); + AsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IntptrTy); + AsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IntptrTy); + AsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt32Ty(), IntptrTy); - AsanHandleNoReturnFunc = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy())); + AsanHandleNoReturnFunc = + M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy()); - AsanPtrCmpFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy)); - AsanPtrSubFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy)); + AsanPtrCmpFunction = + M.getOrInsertFunction(kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy); + AsanPtrSubFunction = + M.getOrInsertFunction(kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy); // We insert an empty inline asm after __asan_report* to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), StringRef(""), StringRef(""), @@ -2400,25 +2514,6 @@ void AddressSanitizer::initializeCallbacks(Module &M) { ArrayType::get(IRB.getInt8Ty(), 0)); } -// virtual -bool AddressSanitizer::doInitialization(Module &M) { - // Initialize the private fields. No one has accessed them before. - GlobalsMD.init(M); - - C = &(M.getContext()); - LongSize = M.getDataLayout().getPointerSizeInBits(); - IntptrTy = Type::getIntNTy(*C, LongSize); - TargetTriple = Triple(M.getTargetTriple()); - - Mapping = getShadowMapping(TargetTriple, LongSize, CompileKernel); - return true; -} - -bool AddressSanitizer::doFinalization(Module &M) { - GlobalsMD.reset(); - return false; -} - bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) { // For each NSObject descendant having a +load method, this method is invoked // by the ObjC runtime before any of the static constructors is called. @@ -2428,7 +2523,7 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) { // We cannot just ignore these methods, because they may call other // instrumented functions. if (F.getName().find(" load]") != std::string::npos) { - Function *AsanInitFunction = + FunctionCallee AsanInitFunction = declareSanitizerInitFunction(*F.getParent(), kAsanInitName, {}); IRBuilder<> IRB(&F.front(), F.front().begin()); IRB.CreateCall(AsanInitFunction, {}); @@ -2460,7 +2555,7 @@ void AddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) { } else { Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal( kAsanShadowMemoryDynamicAddress, IntptrTy); - LocalDynamicShadow = IRB.CreateLoad(GlobalDynamicAddress); + LocalDynamicShadow = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress); } } @@ -2492,7 +2587,8 @@ void AddressSanitizer::markEscapedLocalAllocas(Function &F) { } } -bool AddressSanitizer::runOnFunction(Function &F) { +bool AddressSanitizer::instrumentFunction(Function &F, + const TargetLibraryInfo *TLI) { if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false; if (!ClDebugFunc.empty() && ClDebugFunc == F.getName()) return false; if (F.getName().startswith("__asan_")) return false; @@ -2511,7 +2607,6 @@ bool AddressSanitizer::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n"); initializeCallbacks(*F.getParent()); - DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); FunctionStateRAII CleanupObj(this); @@ -2532,8 +2627,6 @@ bool AddressSanitizer::runOnFunction(Function &F) { bool IsWrite; unsigned Alignment; uint64_t TypeSize; - const TargetLibraryInfo *TLI = - &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); // Fill the set of memory operations to instrument. for (auto &BB : F) { @@ -2557,8 +2650,10 @@ bool AddressSanitizer::runOnFunction(Function &F) { continue; // We've seen this temp in the current BB. } } - } else if (ClInvalidPointerPairs && - isInterestingPointerComparisonOrSubtraction(&Inst)) { + } else if (((ClInvalidPointerPairs || ClInvalidPointerCmp) && + isInterestingPointerComparison(&Inst)) || + ((ClInvalidPointerPairs || ClInvalidPointerSub) && + isInterestingPointerSubtraction(&Inst))) { PointerComparisonsOrSubtracts.push_back(&Inst); continue; } else if (isa<MemIntrinsic>(Inst)) { @@ -2569,7 +2664,8 @@ bool AddressSanitizer::runOnFunction(Function &F) { if (CS) { // A call inside BB. TempsToInstrument.clear(); - if (CS.doesNotReturn()) NoReturnCalls.push_back(CS.getInstruction()); + if (CS.doesNotReturn() && !CS->getMetadata("nosanitize")) + NoReturnCalls.push_back(CS.getInstruction()); } if (CallInst *CI = dyn_cast<CallInst>(&Inst)) maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI); @@ -2606,7 +2702,7 @@ bool AddressSanitizer::runOnFunction(Function &F) { FunctionStackPoisoner FSP(F, *this); bool ChangedStack = FSP.runOnFunction(); - // We must unpoison the stack before every NoReturn call (throw, _exit, etc). + // We must unpoison the stack before NoReturn calls (throw, _exit, etc). // See e.g. https://github.com/google/sanitizers/issues/37 for (auto CI : NoReturnCalls) { IRBuilder<> IRB(CI); @@ -2643,20 +2739,17 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) { IRBuilder<> IRB(*C); for (int i = 0; i <= kMaxAsanStackMallocSizeClass; i++) { std::string Suffix = itostr(i); - AsanStackMallocFunc[i] = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(kAsanStackMallocNameTemplate + Suffix, IntptrTy, - IntptrTy)); - AsanStackFreeFunc[i] = checkSanitizerInterfaceFunction( + AsanStackMallocFunc[i] = M.getOrInsertFunction( + kAsanStackMallocNameTemplate + Suffix, IntptrTy, IntptrTy); + AsanStackFreeFunc[i] = M.getOrInsertFunction(kAsanStackFreeNameTemplate + Suffix, - IRB.getVoidTy(), IntptrTy, IntptrTy)); + IRB.getVoidTy(), IntptrTy, IntptrTy); } if (ASan.UseAfterScope) { - AsanPoisonStackMemoryFunc = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(kAsanPoisonStackMemoryName, IRB.getVoidTy(), - IntptrTy, IntptrTy)); - AsanUnpoisonStackMemoryFunc = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), - IntptrTy, IntptrTy)); + AsanPoisonStackMemoryFunc = M.getOrInsertFunction( + kAsanPoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy); + AsanUnpoisonStackMemoryFunc = M.getOrInsertFunction( + kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy); } for (size_t Val : {0x00, 0xf1, 0xf2, 0xf3, 0xf5, 0xf8}) { @@ -2664,15 +2757,13 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) { Name << kAsanSetShadowPrefix; Name << std::setw(2) << std::setfill('0') << std::hex << Val; AsanSetShadowFunc[Val] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - Name.str(), IRB.getVoidTy(), IntptrTy, IntptrTy)); + M.getOrInsertFunction(Name.str(), IRB.getVoidTy(), IntptrTy, IntptrTy); } - AsanAllocaPoisonFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy)); - AsanAllocasUnpoisonFunc = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy)); + AsanAllocaPoisonFunc = M.getOrInsertFunction( + kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy); + AsanAllocasUnpoisonFunc = M.getOrInsertFunction( + kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy); } void FunctionStackPoisoner::copyToShadowInline(ArrayRef<uint8_t> ShadowMask, @@ -2958,7 +3049,7 @@ void FunctionStackPoisoner::processStaticAllocas() { Value *FakeStack; Value *LocalStackBase; Value *LocalStackBaseAlloca; - bool Deref; + uint8_t DIExprFlags = DIExpression::ApplyOffset; if (DoStackMalloc) { LocalStackBaseAlloca = @@ -2969,9 +3060,9 @@ void FunctionStackPoisoner::processStaticAllocas() { // void *LocalStackBase = (FakeStack) ? FakeStack : alloca(LocalStackSize); Constant *OptionDetectUseAfterReturn = F.getParent()->getOrInsertGlobal( kAsanOptionDetectUseAfterReturn, IRB.getInt32Ty()); - Value *UseAfterReturnIsEnabled = - IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUseAfterReturn), - Constant::getNullValue(IRB.getInt32Ty())); + Value *UseAfterReturnIsEnabled = IRB.CreateICmpNE( + IRB.CreateLoad(IRB.getInt32Ty(), OptionDetectUseAfterReturn), + Constant::getNullValue(IRB.getInt32Ty())); Instruction *Term = SplitBlockAndInsertIfThen(UseAfterReturnIsEnabled, InsBefore, false); IRBuilder<> IRBIf(Term); @@ -2999,7 +3090,7 @@ void FunctionStackPoisoner::processStaticAllocas() { LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack); IRB.SetCurrentDebugLocation(EntryDebugLocation); IRB.CreateStore(LocalStackBase, LocalStackBaseAlloca); - Deref = true; + DIExprFlags |= DIExpression::DerefBefore; } else { // void *FakeStack = nullptr; // void *LocalStackBase = alloca(LocalStackSize); @@ -3007,14 +3098,13 @@ void FunctionStackPoisoner::processStaticAllocas() { LocalStackBase = DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca; LocalStackBaseAlloca = LocalStackBase; - Deref = false; } // Replace Alloca instructions with base+offset. for (const auto &Desc : SVD) { AllocaInst *AI = Desc.AI; - replaceDbgDeclareForAlloca(AI, LocalStackBaseAlloca, DIB, Deref, - Desc.Offset, DIExpression::NoDeref); + replaceDbgDeclareForAlloca(AI, LocalStackBaseAlloca, DIB, DIExprFlags, + Desc.Offset); Value *NewAllocaPtr = IRB.CreateIntToPtr( IRB.CreateAdd(LocalStackBase, ConstantInt::get(IntptrTy, Desc.Offset)), AI->getType()); @@ -3105,7 +3195,7 @@ void FunctionStackPoisoner::processStaticAllocas() { FakeStack, ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8)); Value *SavedFlagPtr = IRBPoison.CreateLoad( - IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy)); + IntptrTy, IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy)); IRBPoison.CreateStore( Constant::getNullValue(IRBPoison.getInt8Ty()), IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getInt8PtrTy())); @@ -3145,41 +3235,6 @@ void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size, // variable may go in and out of scope several times, e.g. in loops). // (3) if we poisoned at least one %alloca in a function, // unpoison the whole stack frame at function exit. - -AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) { - if (AllocaInst *AI = dyn_cast<AllocaInst>(V)) - // We're interested only in allocas we can handle. - return ASan.isInterestingAlloca(*AI) ? AI : nullptr; - // See if we've already calculated (or started to calculate) alloca for a - // given value. - AllocaForValueMapTy::iterator I = AllocaForValue.find(V); - if (I != AllocaForValue.end()) return I->second; - // Store 0 while we're calculating alloca for value V to avoid - // infinite recursion if the value references itself. - AllocaForValue[V] = nullptr; - AllocaInst *Res = nullptr; - if (CastInst *CI = dyn_cast<CastInst>(V)) - Res = findAllocaForValue(CI->getOperand(0)); - else if (PHINode *PN = dyn_cast<PHINode>(V)) { - for (Value *IncValue : PN->incoming_values()) { - // Allow self-referencing phi-nodes. - if (IncValue == PN) continue; - AllocaInst *IncValueAI = findAllocaForValue(IncValue); - // AI for incoming values should exist and should all be equal. - if (IncValueAI == nullptr || (Res != nullptr && IncValueAI != Res)) - return nullptr; - Res = IncValueAI; - } - } else if (GetElementPtrInst *EP = dyn_cast<GetElementPtrInst>(V)) { - Res = findAllocaForValue(EP->getPointerOperand()); - } else { - LLVM_DEBUG(dbgs() << "Alloca search canceled on unknown instruction: " << *V - << "\n"); - } - if (Res) AllocaForValue[V] = Res; - return Res; -} - void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) { IRBuilder<> IRB(AI); diff --git a/lib/Transforms/Instrumentation/BoundsChecking.cpp b/lib/Transforms/Instrumentation/BoundsChecking.cpp index a0c78e0468c61..4dc9b611c1567 100644 --- a/lib/Transforms/Instrumentation/BoundsChecking.cpp +++ b/lib/Transforms/Instrumentation/BoundsChecking.cpp @@ -1,9 +1,8 @@ //===- BoundsChecking.cpp - Instrumentation for run-time bounds checking --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -143,8 +142,9 @@ static void insertBoundsCheck(Value *Or, BuilderTy IRB, GetTrapBBT GetTrapBB) { static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI, ScalarEvolution &SE) { const DataLayout &DL = F.getParent()->getDataLayout(); - ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(), - /*RoundToAlign=*/true); + ObjectSizeOpts EvalOpts; + EvalOpts.RoundToAlign = true; + ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(), EvalOpts); // check HANDLE_MEMORY_INST in include/llvm/Instruction.def for memory // touching instructions diff --git a/lib/Transforms/Instrumentation/CFGMST.h b/lib/Transforms/Instrumentation/CFGMST.h index e178ef386e68f..971e00041762d 100644 --- a/lib/Transforms/Instrumentation/CFGMST.h +++ b/lib/Transforms/Instrumentation/CFGMST.h @@ -1,9 +1,8 @@ //===-- CFGMST.h - Minimum Spanning Tree for CFG ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -196,11 +195,10 @@ public: // Sort CFG edges based on its weight. void sortEdgesByWeight() { - std::stable_sort(AllEdges.begin(), AllEdges.end(), - [](const std::unique_ptr<Edge> &Edge1, - const std::unique_ptr<Edge> &Edge2) { - return Edge1->Weight > Edge2->Weight; - }); + llvm::stable_sort(AllEdges, [](const std::unique_ptr<Edge> &Edge1, + const std::unique_ptr<Edge> &Edge2) { + return Edge1->Weight > Edge2->Weight; + }); } // Traverse all the edges and compute the Minimum Weight Spanning Tree diff --git a/lib/Transforms/Instrumentation/CGProfile.cpp b/lib/Transforms/Instrumentation/CGProfile.cpp index cdcd017269061..358abab3cceb7 100644 --- a/lib/Transforms/Instrumentation/CGProfile.cpp +++ b/lib/Transforms/Instrumentation/CGProfile.cpp @@ -1,9 +1,8 @@ //===-- CGProfile.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/lib/Transforms/Instrumentation/ControlHeightReduction.cpp index 1ada0b7130927..3f4f9bc7145dd 100644 --- a/lib/Transforms/Instrumentation/ControlHeightReduction.cpp +++ b/lib/Transforms/Instrumentation/ControlHeightReduction.cpp @@ -1,9 +1,8 @@ //===-- ControlHeightReduction.cpp - Control Height Reduction -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -547,19 +546,25 @@ static std::set<Value *> getBaseValues(Value *V, static bool checkHoistValue(Value *V, Instruction *InsertPoint, DominatorTree &DT, DenseSet<Instruction *> &Unhoistables, - DenseSet<Instruction *> *HoistStops) { + DenseSet<Instruction *> *HoistStops, + DenseMap<Instruction *, bool> &Visited) { assert(InsertPoint && "Null InsertPoint"); if (auto *I = dyn_cast<Instruction>(V)) { + if (Visited.count(I)) { + return Visited[I]; + } assert(DT.getNode(I->getParent()) && "DT must contain I's parent block"); assert(DT.getNode(InsertPoint->getParent()) && "DT must contain Destination"); if (Unhoistables.count(I)) { // Don't hoist if they are not to be hoisted. + Visited[I] = false; return false; } if (DT.dominates(I, InsertPoint)) { // We are already above the insert point. Stop here. if (HoistStops) HoistStops->insert(I); + Visited[I] = true; return true; } // We aren't not above the insert point, check if we can hoist it above the @@ -569,7 +574,8 @@ checkHoistValue(Value *V, Instruction *InsertPoint, DominatorTree &DT, DenseSet<Instruction *> OpsHoistStops; bool AllOpsHoisted = true; for (Value *Op : I->operands()) { - if (!checkHoistValue(Op, InsertPoint, DT, Unhoistables, &OpsHoistStops)) { + if (!checkHoistValue(Op, InsertPoint, DT, Unhoistables, &OpsHoistStops, + Visited)) { AllOpsHoisted = false; break; } @@ -578,9 +584,11 @@ checkHoistValue(Value *V, Instruction *InsertPoint, DominatorTree &DT, CHR_DEBUG(dbgs() << "checkHoistValue " << *I << "\n"); if (HoistStops) HoistStops->insert(OpsHoistStops.begin(), OpsHoistStops.end()); + Visited[I] = true; return true; } } + Visited[I] = false; return false; } // Non-instructions are considered hoistable. @@ -893,8 +901,9 @@ void CHR::checkScopeHoistable(CHRScope *Scope) { ++it; continue; } + DenseMap<Instruction *, bool> Visited; bool IsHoistable = checkHoistValue(SI->getCondition(), InsertPoint, - DT, Unhoistables, nullptr); + DT, Unhoistables, nullptr, Visited); if (!IsHoistable) { CHR_DEBUG(dbgs() << "Dropping select " << *SI << "\n"); ORE.emit([&]() { @@ -913,8 +922,9 @@ void CHR::checkScopeHoistable(CHRScope *Scope) { InsertPoint = getBranchInsertPoint(RI); CHR_DEBUG(dbgs() << "InsertPoint " << *InsertPoint << "\n"); if (RI.HasBranch && InsertPoint != Branch) { + DenseMap<Instruction *, bool> Visited; bool IsHoistable = checkHoistValue(Branch->getCondition(), InsertPoint, - DT, Unhoistables, nullptr); + DT, Unhoistables, nullptr, Visited); if (!IsHoistable) { // If the branch isn't hoistable, drop the selects in the entry // block, preferring the branch, which makes the branch the hoist @@ -945,15 +955,17 @@ void CHR::checkScopeHoistable(CHRScope *Scope) { if (RI.HasBranch) { assert(!DT.dominates(Branch, InsertPoint) && "Branch can't be already above the hoist point"); + DenseMap<Instruction *, bool> Visited; assert(checkHoistValue(Branch->getCondition(), InsertPoint, - DT, Unhoistables, nullptr) && + DT, Unhoistables, nullptr, Visited) && "checkHoistValue for branch"); } for (auto *SI : Selects) { assert(!DT.dominates(SI, InsertPoint) && "SI can't be already above the hoist point"); + DenseMap<Instruction *, bool> Visited; assert(checkHoistValue(SI->getCondition(), InsertPoint, DT, - Unhoistables, nullptr) && + Unhoistables, nullptr, Visited) && "checkHoistValue for selects"); } CHR_DEBUG(dbgs() << "Result\n"); @@ -1054,7 +1066,8 @@ static bool shouldSplit(Instruction *InsertPoint, assert(InsertPoint && "Null InsertPoint"); // If any of Bases isn't hoistable to the hoist point, split. for (Value *V : ConditionValues) { - if (!checkHoistValue(V, InsertPoint, DT, Unhoistables, nullptr)) { + DenseMap<Instruction *, bool> Visited; + if (!checkHoistValue(V, InsertPoint, DT, Unhoistables, nullptr, Visited)) { CHR_DEBUG(dbgs() << "Split. checkHoistValue false " << *V << "\n"); return true; // Not hoistable, split. } @@ -1383,8 +1396,9 @@ void CHR::setCHRRegions(CHRScope *Scope, CHRScope *OutermostScope) { "Must be truthy or falsy"); auto *BI = cast<BranchInst>(R->getEntry()->getTerminator()); // Note checkHoistValue fills in HoistStops. + DenseMap<Instruction *, bool> Visited; bool IsHoistable = checkHoistValue(BI->getCondition(), InsertPoint, DT, - Unhoistables, &HoistStops); + Unhoistables, &HoistStops, Visited); assert(IsHoistable && "Must be hoistable"); (void)(IsHoistable); // Unused in release build IsHoisted = true; @@ -1394,8 +1408,9 @@ void CHR::setCHRRegions(CHRScope *Scope, CHRScope *OutermostScope) { OutermostScope->FalseBiasedSelects.count(SI) > 0) && "Must be true or false biased"); // Note checkHoistValue fills in HoistStops. + DenseMap<Instruction *, bool> Visited; bool IsHoistable = checkHoistValue(SI->getCondition(), InsertPoint, DT, - Unhoistables, &HoistStops); + Unhoistables, &HoistStops, Visited); assert(IsHoistable && "Must be hoistable"); (void)(IsHoistable); // Unused in release build IsHoisted = true; @@ -1417,7 +1432,7 @@ void CHR::sortScopes(SmallVectorImpl<CHRScope *> &Input, SmallVectorImpl<CHRScope *> &Output) { Output.resize(Input.size()); llvm::copy(Input, Output.begin()); - std::stable_sort(Output.begin(), Output.end(), CHRScopeSorter); + llvm::stable_sort(Output, CHRScopeSorter); } // Return true if V is already hoisted or was hoisted (along with its operands) @@ -1425,7 +1440,8 @@ void CHR::sortScopes(SmallVectorImpl<CHRScope *> &Input, static void hoistValue(Value *V, Instruction *HoistPoint, Region *R, HoistStopMapTy &HoistStopMap, DenseSet<Instruction *> &HoistedSet, - DenseSet<PHINode *> &TrivialPHIs) { + DenseSet<PHINode *> &TrivialPHIs, + DominatorTree &DT) { auto IT = HoistStopMap.find(R); assert(IT != HoistStopMap.end() && "Region must be in hoist stop map"); DenseSet<Instruction *> &HoistStops = IT->second; @@ -1445,8 +1461,21 @@ static void hoistValue(Value *V, Instruction *HoistPoint, Region *R, // Already hoisted, return. return; assert(isHoistableInstructionType(I) && "Unhoistable instruction type"); + assert(DT.getNode(I->getParent()) && "DT must contain I's block"); + assert(DT.getNode(HoistPoint->getParent()) && + "DT must contain HoistPoint block"); + if (DT.dominates(I, HoistPoint)) + // We are already above the hoist point. Stop here. This may be necessary + // when multiple scopes would independently hoist the same + // instruction. Since an outer (dominating) scope would hoist it to its + // entry before an inner (dominated) scope would to its entry, the inner + // scope may see the instruction already hoisted, in which case it + // potentially wrong for the inner scope to hoist it and could cause bad + // IR (non-dominating def), but safe to skip hoisting it instead because + // it's already in a block that dominates the inner scope. + return; for (Value *Op : I->operands()) { - hoistValue(Op, HoistPoint, R, HoistStopMap, HoistedSet, TrivialPHIs); + hoistValue(Op, HoistPoint, R, HoistStopMap, HoistedSet, TrivialPHIs, DT); } I->moveBefore(HoistPoint); HoistedSet.insert(I); @@ -1457,7 +1486,8 @@ static void hoistValue(Value *V, Instruction *HoistPoint, Region *R, // Hoist the dependent condition values of the branches and the selects in the // scope to the insert point. static void hoistScopeConditions(CHRScope *Scope, Instruction *HoistPoint, - DenseSet<PHINode *> &TrivialPHIs) { + DenseSet<PHINode *> &TrivialPHIs, + DominatorTree &DT) { DenseSet<Instruction *> HoistedSet; for (const RegInfo &RI : Scope->CHRRegions) { Region *R = RI.R; @@ -1466,7 +1496,7 @@ static void hoistScopeConditions(CHRScope *Scope, Instruction *HoistPoint, if (RI.HasBranch && (IsTrueBiased || IsFalseBiased)) { auto *BI = cast<BranchInst>(R->getEntry()->getTerminator()); hoistValue(BI->getCondition(), HoistPoint, R, Scope->HoistStopMap, - HoistedSet, TrivialPHIs); + HoistedSet, TrivialPHIs, DT); } for (SelectInst *SI : RI.Selects) { bool IsTrueBiased = Scope->TrueBiasedSelects.count(SI); @@ -1474,7 +1504,7 @@ static void hoistScopeConditions(CHRScope *Scope, Instruction *HoistPoint, if (!(IsTrueBiased || IsFalseBiased)) continue; hoistValue(SI->getCondition(), HoistPoint, R, Scope->HoistStopMap, - HoistedSet, TrivialPHIs); + HoistedSet, TrivialPHIs, DT); } } } @@ -1708,7 +1738,7 @@ void CHR::transformScopes(CHRScope *Scope, DenseSet<PHINode *> &TrivialPHIs) { #endif // Hoist the conditional values of the branches/selects. - hoistScopeConditions(Scope, PreEntryBlock->getTerminator(), TrivialPHIs); + hoistScopeConditions(Scope, PreEntryBlock->getTerminator(), TrivialPHIs, DT); #ifndef NDEBUG assertBranchOrSelectConditionHoisted(Scope, PreEntryBlock); diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index 4c3c6c9added0..2279c1bcb6a89 100644 --- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -1,9 +1,8 @@ //===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -333,6 +332,8 @@ class DataFlowSanitizer : public ModulePass { Constant *RetvalTLS; void *(*GetArgTLSPtr)(); void *(*GetRetvalTLSPtr)(); + FunctionType *GetArgTLSTy; + FunctionType *GetRetvalTLSTy; Constant *GetArgTLS; Constant *GetRetvalTLS; Constant *ExternalShadowMask; @@ -342,13 +343,13 @@ class DataFlowSanitizer : public ModulePass { FunctionType *DFSanSetLabelFnTy; FunctionType *DFSanNonzeroLabelFnTy; FunctionType *DFSanVarargWrapperFnTy; - Constant *DFSanUnionFn; - Constant *DFSanCheckedUnionFn; - Constant *DFSanUnionLoadFn; - Constant *DFSanUnimplementedFn; - Constant *DFSanSetLabelFn; - Constant *DFSanNonzeroLabelFn; - Constant *DFSanVarargWrapperFn; + FunctionCallee DFSanUnionFn; + FunctionCallee DFSanCheckedUnionFn; + FunctionCallee DFSanUnionLoadFn; + FunctionCallee DFSanUnimplementedFn; + FunctionCallee DFSanSetLabelFn; + FunctionCallee DFSanNonzeroLabelFn; + FunctionCallee DFSanVarargWrapperFn; MDNode *ColdCallWeights; DFSanABIList ABIList; DenseMap<Value *, Function *> UnwrappedFnMap; @@ -436,6 +437,7 @@ public: } void visitOperandShadowInst(Instruction &I); + void visitUnaryOperator(UnaryOperator &UO); void visitBinaryOperator(BinaryOperator &BO); void visitCastInst(CastInst &CI); void visitCmpInst(CmpInst &CI); @@ -581,17 +583,17 @@ bool DataFlowSanitizer::doInitialization(Module &M) { if (GetArgTLSPtr) { Type *ArgTLSTy = ArrayType::get(ShadowTy, 64); ArgTLS = nullptr; + GetArgTLSTy = FunctionType::get(PointerType::getUnqual(ArgTLSTy), false); GetArgTLS = ConstantExpr::getIntToPtr( ConstantInt::get(IntptrTy, uintptr_t(GetArgTLSPtr)), - PointerType::getUnqual( - FunctionType::get(PointerType::getUnqual(ArgTLSTy), false))); + PointerType::getUnqual(GetArgTLSTy)); } if (GetRetvalTLSPtr) { RetvalTLS = nullptr; + GetRetvalTLSTy = FunctionType::get(PointerType::getUnqual(ShadowTy), false); GetRetvalTLS = ConstantExpr::getIntToPtr( ConstantInt::get(IntptrTy, uintptr_t(GetRetvalTLSPtr)), - PointerType::getUnqual( - FunctionType::get(PointerType::getUnqual(ShadowTy), false))); + PointerType::getUnqual(GetRetvalTLSTy)); } ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000); @@ -678,8 +680,8 @@ DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName, Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT, StringRef FName) { FunctionType *FTT = getTrampolineFunctionType(FT); - Constant *C = Mod->getOrInsertFunction(FName, FTT); - Function *F = dyn_cast<Function>(C); + FunctionCallee C = Mod->getOrInsertFunction(FName, FTT); + Function *F = dyn_cast<Function>(C.getCallee()); if (F && F->isDeclaration()) { F->setLinkage(GlobalValue::LinkOnceODRLinkage); BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F); @@ -687,7 +689,7 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT, Function::arg_iterator AI = F->arg_begin(); ++AI; for (unsigned N = FT->getNumParams(); N != 0; ++AI, --N) Args.push_back(&*AI); - CallInst *CI = CallInst::Create(&*F->arg_begin(), Args, "", BB); + CallInst *CI = CallInst::Create(FT, &*F->arg_begin(), Args, "", BB); ReturnInst *RI; if (FT->getReturnType()->isVoidTy()) RI = ReturnInst::Create(*Ctx, BB); @@ -704,7 +706,7 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT, &*std::prev(F->arg_end()), RI); } - return C; + return cast<Constant>(C.getCallee()); } bool DataFlowSanitizer::runOnModule(Module &M) { @@ -726,35 +728,51 @@ bool DataFlowSanitizer::runOnModule(Module &M) { ExternalShadowMask = Mod->getOrInsertGlobal(kDFSanExternShadowPtrMask, IntptrTy); - DFSanUnionFn = Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy); - if (Function *F = dyn_cast<Function>(DFSanUnionFn)) { - F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); - F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone); - F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); - F->addParamAttr(0, Attribute::ZExt); - F->addParamAttr(1, Attribute::ZExt); + { + AttributeList AL; + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::NoUnwind); + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::ReadNone); + AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex, + Attribute::ZExt); + AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt); + AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt); + DFSanUnionFn = + Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy, AL); } - DFSanCheckedUnionFn = Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy); - if (Function *F = dyn_cast<Function>(DFSanCheckedUnionFn)) { - F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); - F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone); - F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); - F->addParamAttr(0, Attribute::ZExt); - F->addParamAttr(1, Attribute::ZExt); + + { + AttributeList AL; + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::NoUnwind); + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::ReadNone); + AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex, + Attribute::ZExt); + AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt); + AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt); + DFSanCheckedUnionFn = + Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy, AL); } - DFSanUnionLoadFn = - Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy); - if (Function *F = dyn_cast<Function>(DFSanUnionLoadFn)) { - F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind); - F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly); - F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); + { + AttributeList AL; + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::NoUnwind); + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::ReadOnly); + AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex, + Attribute::ZExt); + DFSanUnionLoadFn = + Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL); } DFSanUnimplementedFn = Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy); - DFSanSetLabelFn = - Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy); - if (Function *F = dyn_cast<Function>(DFSanSetLabelFn)) { - F->addParamAttr(0, Attribute::ZExt); + { + AttributeList AL; + AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt); + DFSanSetLabelFn = + Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL); } DFSanNonzeroLabelFn = Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy); @@ -765,13 +783,13 @@ bool DataFlowSanitizer::runOnModule(Module &M) { SmallPtrSet<Function *, 2> FnsWithNativeABI; for (Function &i : M) { if (!i.isIntrinsic() && - &i != DFSanUnionFn && - &i != DFSanCheckedUnionFn && - &i != DFSanUnionLoadFn && - &i != DFSanUnimplementedFn && - &i != DFSanSetLabelFn && - &i != DFSanNonzeroLabelFn && - &i != DFSanVarargWrapperFn) + &i != DFSanUnionFn.getCallee()->stripPointerCasts() && + &i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() && + &i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() && + &i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() && + &i != DFSanSetLabelFn.getCallee()->stripPointerCasts() && + &i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() && + &i != DFSanVarargWrapperFn.getCallee()->stripPointerCasts()) FnsToInstrument.push_back(&i); } @@ -982,7 +1000,7 @@ Value *DFSanFunction::getArgTLSPtr() { return ArgTLSPtr = DFS.ArgTLS; IRBuilder<> IRB(&F->getEntryBlock().front()); - return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLS, {}); + return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLSTy, DFS.GetArgTLS, {}); } Value *DFSanFunction::getRetvalTLS() { @@ -992,12 +1010,14 @@ Value *DFSanFunction::getRetvalTLS() { return RetvalTLSPtr = DFS.RetvalTLS; IRBuilder<> IRB(&F->getEntryBlock().front()); - return RetvalTLSPtr = IRB.CreateCall(DFS.GetRetvalTLS, {}); + return RetvalTLSPtr = + IRB.CreateCall(DFS.GetRetvalTLSTy, DFS.GetRetvalTLS, {}); } Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) { IRBuilder<> IRB(Pos); - return IRB.CreateConstGEP2_64(getArgTLSPtr(), 0, Idx); + return IRB.CreateConstGEP2_64(ArrayType::get(DFS.ShadowTy, 64), + getArgTLSPtr(), 0, Idx); } Value *DFSanFunction::getShadow(Value *V) { @@ -1015,7 +1035,8 @@ Value *DFSanFunction::getShadow(Value *V) { DFS.ArgTLS ? &*F->getEntryBlock().begin() : cast<Instruction>(ArgTLSPtr)->getNextNode(); IRBuilder<> IRB(ArgTLSPos); - Shadow = IRB.CreateLoad(getArgTLS(A->getArgNo(), ArgTLSPos)); + Shadow = + IRB.CreateLoad(DFS.ShadowTy, getArgTLS(A->getArgNo(), ArgTLSPos)); break; } case DataFlowSanitizer::IA_Args: { @@ -1165,15 +1186,15 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, const auto i = AllocaShadowMap.find(AI); if (i != AllocaShadowMap.end()) { IRBuilder<> IRB(Pos); - return IRB.CreateLoad(i->second); + return IRB.CreateLoad(DFS.ShadowTy, i->second); } } uint64_t ShadowAlign = Align * DFS.ShadowWidth / 8; - SmallVector<Value *, 2> Objs; + SmallVector<const Value *, 2> Objs; GetUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout()); bool AllConstants = true; - for (Value *Obj : Objs) { + for (const Value *Obj : Objs) { if (isa<Function>(Obj) || isa<BlockAddress>(Obj)) continue; if (isa<GlobalVariable>(Obj) && cast<GlobalVariable>(Obj)->isConstant()) @@ -1190,7 +1211,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, case 0: return DFS.ZeroShadow; case 1: { - LoadInst *LI = new LoadInst(ShadowAddr, "", Pos); + LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos); LI->setAlignment(ShadowAlign); return LI; } @@ -1198,8 +1219,9 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, IRBuilder<> IRB(Pos); Value *ShadowAddr1 = IRB.CreateGEP(DFS.ShadowTy, ShadowAddr, ConstantInt::get(DFS.IntptrTy, 1)); - return combineShadows(IRB.CreateAlignedLoad(ShadowAddr, ShadowAlign), - IRB.CreateAlignedLoad(ShadowAddr1, ShadowAlign), Pos); + return combineShadows( + IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr, ShadowAlign), + IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos); } } if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidth) == 0) { @@ -1218,7 +1240,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, IRBuilder<> IRB(Pos); Value *WideAddr = IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx)); - Value *WideShadow = IRB.CreateAlignedLoad(WideAddr, ShadowAlign); + Value *WideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.ShadowTy); Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidth); Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidth); @@ -1251,7 +1274,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, IRBuilder<> NextIRB(NextBB); WideAddr = NextIRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr, ConstantInt::get(DFS.IntptrTy, 1)); - Value *NextWideShadow = NextIRB.CreateAlignedLoad(WideAddr, ShadowAlign); + Value *NextWideShadow = NextIRB.CreateAlignedLoad(NextIRB.getInt64Ty(), + WideAddr, ShadowAlign); ShadowsEq = NextIRB.CreateICmpEQ(WideShadow, NextWideShadow); LastBr->setSuccessor(0, NextBB); LastBr = NextIRB.CreateCondBr(ShadowsEq, FallbackBB, FallbackBB); @@ -1375,6 +1399,10 @@ void DFSanVisitor::visitStoreInst(StoreInst &SI) { DFSF.storeShadow(SI.getPointerOperand(), Size, Align, Shadow, &SI); } +void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) { + visitOperandShadowInst(UO); +} + void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) { visitOperandShadowInst(BO); } @@ -1470,7 +1498,7 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) { DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr); SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr); auto *MTI = cast<MemTransferInst>( - IRB.CreateCall(I.getCalledValue(), + IRB.CreateCall(I.getFunctionType(), I.getCalledValue(), {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()})); if (ClPreserveAlignment) { MTI->setDestAlignment(I.getDestAlignment() * (DFSF.DFS.ShadowWidth / 8)); @@ -1513,7 +1541,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) { // Calls to this function are synthesized in wrappers, and we shouldn't // instrument them. - if (F == DFSF.DFS.DFSanVarargWrapperFn) + if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts()) return; IRBuilder<> IRB(CS.getInstruction()); @@ -1546,9 +1574,9 @@ void DFSanVisitor::visitCallSite(CallSite CS) { TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT); std::string CustomFName = "__dfsw_"; CustomFName += F->getName(); - Constant *CustomF = DFSF.DFS.Mod->getOrInsertFunction( + FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction( CustomFName, CustomFn.TransformedType); - if (Function *CustomFn = dyn_cast<Function>(CustomF)) { + if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) { CustomFn->copyAttributesFrom(F); // Custom functions returning non-void will write to the return label. @@ -1628,7 +1656,8 @@ void DFSanVisitor::visitCallSite(CallSite CS) { } if (!FT->getReturnType()->isVoidTy()) { - LoadInst *LabelLoad = IRB.CreateLoad(DFSF.LabelReturnAlloca); + LoadInst *LabelLoad = + IRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.LabelReturnAlloca); DFSF.setShadow(CustomCI, LabelLoad); } @@ -1666,7 +1695,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) { if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) { IRBuilder<> NextIRB(Next); - LoadInst *LI = NextIRB.CreateLoad(DFSF.getRetvalTLS()); + LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.getRetvalTLS()); DFSF.SkipInsts.insert(LI); DFSF.setShadow(CS.getInstruction(), LI); DFSF.NonZeroChecks.push_back(LI); @@ -1706,10 +1735,10 @@ void DFSanVisitor::visitCallSite(CallSite CS) { CallSite NewCS; if (InvokeInst *II = dyn_cast<InvokeInst>(CS.getInstruction())) { - NewCS = IRB.CreateInvoke(Func, II->getNormalDest(), II->getUnwindDest(), - Args); + NewCS = IRB.CreateInvoke(NewFT, Func, II->getNormalDest(), + II->getUnwindDest(), Args); } else { - NewCS = IRB.CreateCall(Func, Args); + NewCS = IRB.CreateCall(NewFT, Func, Args); } NewCS.setCallingConv(CS.getCallingConv()); NewCS.setAttributes(CS.getAttributes().removeAttributes( diff --git a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp b/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp deleted file mode 100644 index db438e78ded95..0000000000000 --- a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp +++ /dev/null @@ -1,900 +0,0 @@ -//===-- EfficiencySanitizer.cpp - performance tuner -----------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of EfficiencySanitizer, a family of performance tuners -// that detects multiple performance issues via separate sub-tools. -// -// The instrumentation phase is straightforward: -// - Take action on every memory access: either inlined instrumentation, -// or Inserted calls to our run-time library. -// - Optimizations may apply to avoid instrumenting some of the accesses. -// - Turn mem{set,cpy,move} instrinsics into library calls. -// The rest is handled by the run-time library. -//===----------------------------------------------------------------------===// - -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Transforms/Utils/Local.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Type.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include "llvm/Transforms/Utils/ModuleUtils.h" - -using namespace llvm; - -#define DEBUG_TYPE "esan" - -// The tool type must be just one of these ClTool* options, as the tools -// cannot be combined due to shadow memory constraints. -static cl::opt<bool> - ClToolCacheFrag("esan-cache-frag", cl::init(false), - cl::desc("Detect data cache fragmentation"), cl::Hidden); -static cl::opt<bool> - ClToolWorkingSet("esan-working-set", cl::init(false), - cl::desc("Measure the working set size"), cl::Hidden); -// Each new tool will get its own opt flag here. -// These are converted to EfficiencySanitizerOptions for use -// in the code. - -static cl::opt<bool> ClInstrumentLoadsAndStores( - "esan-instrument-loads-and-stores", cl::init(true), - cl::desc("Instrument loads and stores"), cl::Hidden); -static cl::opt<bool> ClInstrumentMemIntrinsics( - "esan-instrument-memintrinsics", cl::init(true), - cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); -static cl::opt<bool> ClInstrumentFastpath( - "esan-instrument-fastpath", cl::init(true), - cl::desc("Instrument fastpath"), cl::Hidden); -static cl::opt<bool> ClAuxFieldInfo( - "esan-aux-field-info", cl::init(true), - cl::desc("Generate binary with auxiliary struct field information"), - cl::Hidden); - -// Experiments show that the performance difference can be 2x or more, -// and accuracy loss is typically negligible, so we turn this on by default. -static cl::opt<bool> ClAssumeIntraCacheLine( - "esan-assume-intra-cache-line", cl::init(true), - cl::desc("Assume each memory access touches just one cache line, for " - "better performance but with a potential loss of accuracy."), - cl::Hidden); - -STATISTIC(NumInstrumentedLoads, "Number of instrumented loads"); -STATISTIC(NumInstrumentedStores, "Number of instrumented stores"); -STATISTIC(NumFastpaths, "Number of instrumented fastpaths"); -STATISTIC(NumAccessesWithIrregularSize, - "Number of accesses with a size outside our targeted callout sizes"); -STATISTIC(NumIgnoredStructs, "Number of ignored structs"); -STATISTIC(NumIgnoredGEPs, "Number of ignored GEP instructions"); -STATISTIC(NumInstrumentedGEPs, "Number of instrumented GEP instructions"); -STATISTIC(NumAssumedIntraCacheLine, - "Number of accesses assumed to be intra-cache-line"); - -static const uint64_t EsanCtorAndDtorPriority = 0; -static const char *const EsanModuleCtorName = "esan.module_ctor"; -static const char *const EsanModuleDtorName = "esan.module_dtor"; -static const char *const EsanInitName = "__esan_init"; -static const char *const EsanExitName = "__esan_exit"; - -// We need to specify the tool to the runtime earlier than -// the ctor is called in some cases, so we set a global variable. -static const char *const EsanWhichToolName = "__esan_which_tool"; - -// We must keep these Shadow* constants consistent with the esan runtime. -// FIXME: Try to place these shadow constants, the names of the __esan_* -// interface functions, and the ToolType enum into a header shared between -// llvm and compiler-rt. -struct ShadowMemoryParams { - uint64_t ShadowMask; - uint64_t ShadowOffs[3]; -}; - -static const ShadowMemoryParams ShadowParams47 = { - 0x00000fffffffffffull, - { - 0x0000130000000000ull, 0x0000220000000000ull, 0x0000440000000000ull, - }}; - -static const ShadowMemoryParams ShadowParams40 = { - 0x0fffffffffull, - { - 0x1300000000ull, 0x2200000000ull, 0x4400000000ull, - }}; - -// This array is indexed by the ToolType enum. -static const int ShadowScale[] = { - 0, // ESAN_None. - 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2. - 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6. -}; - -// MaxStructCounterNameSize is a soft size limit to avoid insanely long -// names for those extremely large structs. -static const unsigned MaxStructCounterNameSize = 512; - -namespace { - -static EfficiencySanitizerOptions -OverrideOptionsFromCL(EfficiencySanitizerOptions Options) { - if (ClToolCacheFrag) - Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag; - else if (ClToolWorkingSet) - Options.ToolType = EfficiencySanitizerOptions::ESAN_WorkingSet; - - // Direct opt invocation with no params will have the default ESAN_None. - // We run the default tool in that case. - if (Options.ToolType == EfficiencySanitizerOptions::ESAN_None) - Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag; - - return Options; -} - -/// EfficiencySanitizer: instrument each module to find performance issues. -class EfficiencySanitizer : public ModulePass { -public: - EfficiencySanitizer( - const EfficiencySanitizerOptions &Opts = EfficiencySanitizerOptions()) - : ModulePass(ID), Options(OverrideOptionsFromCL(Opts)) {} - StringRef getPassName() const override; - void getAnalysisUsage(AnalysisUsage &AU) const override; - bool runOnModule(Module &M) override; - static char ID; - -private: - bool initOnModule(Module &M); - void initializeCallbacks(Module &M); - bool shouldIgnoreStructType(StructType *StructTy); - void createStructCounterName( - StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr); - void createCacheFragAuxGV( - Module &M, const DataLayout &DL, StructType *StructTy, - GlobalVariable *&TypeNames, GlobalVariable *&Offsets, GlobalVariable *&Size); - GlobalVariable *createCacheFragInfoGV(Module &M, const DataLayout &DL, - Constant *UnitName); - Constant *createEsanInitToolInfoArg(Module &M, const DataLayout &DL); - void createDestructor(Module &M, Constant *ToolInfoArg); - bool runOnFunction(Function &F, Module &M); - bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); - bool instrumentMemIntrinsic(MemIntrinsic *MI); - bool instrumentGetElementPtr(Instruction *I, Module &M); - bool insertCounterUpdate(Instruction *I, StructType *StructTy, - unsigned CounterIdx); - unsigned getFieldCounterIdx(StructType *StructTy) { - return 0; - } - unsigned getArrayCounterIdx(StructType *StructTy) { - return StructTy->getNumElements(); - } - unsigned getStructCounterSize(StructType *StructTy) { - // The struct counter array includes: - // - one counter for each struct field, - // - one counter for the struct access within an array. - return (StructTy->getNumElements()/*field*/ + 1/*array*/); - } - bool shouldIgnoreMemoryAccess(Instruction *I); - int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); - Value *appToShadow(Value *Shadow, IRBuilder<> &IRB); - bool instrumentFastpath(Instruction *I, const DataLayout &DL, bool IsStore, - Value *Addr, unsigned Alignment); - // Each tool has its own fastpath routine: - bool instrumentFastpathCacheFrag(Instruction *I, const DataLayout &DL, - Value *Addr, unsigned Alignment); - bool instrumentFastpathWorkingSet(Instruction *I, const DataLayout &DL, - Value *Addr, unsigned Alignment); - - EfficiencySanitizerOptions Options; - LLVMContext *Ctx; - Type *IntptrTy; - // Our slowpath involves callouts to the runtime library. - // Access sizes are powers of two: 1, 2, 4, 8, 16. - static const size_t NumberOfAccessSizes = 5; - Function *EsanAlignedLoad[NumberOfAccessSizes]; - Function *EsanAlignedStore[NumberOfAccessSizes]; - Function *EsanUnalignedLoad[NumberOfAccessSizes]; - Function *EsanUnalignedStore[NumberOfAccessSizes]; - // For irregular sizes of any alignment: - Function *EsanUnalignedLoadN, *EsanUnalignedStoreN; - Function *MemmoveFn, *MemcpyFn, *MemsetFn; - Function *EsanCtorFunction; - Function *EsanDtorFunction; - // Remember the counter variable for each struct type to avoid - // recomputing the variable name later during instrumentation. - std::map<Type *, GlobalVariable *> StructTyMap; - ShadowMemoryParams ShadowParams; -}; -} // namespace - -char EfficiencySanitizer::ID = 0; -INITIALIZE_PASS_BEGIN( - EfficiencySanitizer, "esan", - "EfficiencySanitizer: finds performance issues.", false, false) -INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) -INITIALIZE_PASS_END( - EfficiencySanitizer, "esan", - "EfficiencySanitizer: finds performance issues.", false, false) - -StringRef EfficiencySanitizer::getPassName() const { - return "EfficiencySanitizer"; -} - -void EfficiencySanitizer::getAnalysisUsage(AnalysisUsage &AU) const { - AU.addRequired<TargetLibraryInfoWrapperPass>(); -} - -ModulePass * -llvm::createEfficiencySanitizerPass(const EfficiencySanitizerOptions &Options) { - return new EfficiencySanitizer(Options); -} - -void EfficiencySanitizer::initializeCallbacks(Module &M) { - IRBuilder<> IRB(M.getContext()); - // Initialize the callbacks. - for (size_t Idx = 0; Idx < NumberOfAccessSizes; ++Idx) { - const unsigned ByteSize = 1U << Idx; - std::string ByteSizeStr = utostr(ByteSize); - // We'll inline the most common (i.e., aligned and frequent sizes) - // load + store instrumentation: these callouts are for the slowpath. - SmallString<32> AlignedLoadName("__esan_aligned_load" + ByteSizeStr); - EsanAlignedLoad[Idx] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - AlignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy())); - SmallString<32> AlignedStoreName("__esan_aligned_store" + ByteSizeStr); - EsanAlignedStore[Idx] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - AlignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy())); - SmallString<32> UnalignedLoadName("__esan_unaligned_load" + ByteSizeStr); - EsanUnalignedLoad[Idx] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - UnalignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy())); - SmallString<32> UnalignedStoreName("__esan_unaligned_store" + ByteSizeStr); - EsanUnalignedStore[Idx] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - UnalignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy())); - } - EsanUnalignedLoadN = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("__esan_unaligned_loadN", IRB.getVoidTy(), - IRB.getInt8PtrTy(), IntptrTy)); - EsanUnalignedStoreN = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("__esan_unaligned_storeN", IRB.getVoidTy(), - IRB.getInt8PtrTy(), IntptrTy)); - MemmoveFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IntptrTy)); - MemcpyFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IntptrTy)); - MemsetFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt32Ty(), IntptrTy)); -} - -bool EfficiencySanitizer::shouldIgnoreStructType(StructType *StructTy) { - if (StructTy == nullptr || StructTy->isOpaque() /* no struct body */) - return true; - return false; -} - -void EfficiencySanitizer::createStructCounterName( - StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr) { - // Append NumFields and field type ids to avoid struct conflicts - // with the same name but different fields. - if (StructTy->hasName()) - NameStr += StructTy->getName(); - else - NameStr += "struct.anon"; - // We allow the actual size of the StructCounterName to be larger than - // MaxStructCounterNameSize and append $NumFields and at least one - // field type id. - // Append $NumFields. - NameStr += "$"; - Twine(StructTy->getNumElements()).toVector(NameStr); - // Append struct field type ids in the reverse order. - for (int i = StructTy->getNumElements() - 1; i >= 0; --i) { - NameStr += "$"; - Twine(StructTy->getElementType(i)->getTypeID()).toVector(NameStr); - if (NameStr.size() >= MaxStructCounterNameSize) - break; - } - if (StructTy->isLiteral()) { - // End with $ for literal struct. - NameStr += "$"; - } -} - -// Create global variables with auxiliary information (e.g., struct field size, -// offset, and type name) for better user report. -void EfficiencySanitizer::createCacheFragAuxGV( - Module &M, const DataLayout &DL, StructType *StructTy, - GlobalVariable *&TypeName, GlobalVariable *&Offset, - GlobalVariable *&Size) { - auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); - auto *Int32Ty = Type::getInt32Ty(*Ctx); - // FieldTypeName. - auto *TypeNameArrayTy = ArrayType::get(Int8PtrTy, StructTy->getNumElements()); - TypeName = new GlobalVariable(M, TypeNameArrayTy, true, - GlobalVariable::InternalLinkage, nullptr); - SmallVector<Constant *, 16> TypeNameVec; - // FieldOffset. - auto *OffsetArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements()); - Offset = new GlobalVariable(M, OffsetArrayTy, true, - GlobalVariable::InternalLinkage, nullptr); - SmallVector<Constant *, 16> OffsetVec; - // FieldSize - auto *SizeArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements()); - Size = new GlobalVariable(M, SizeArrayTy, true, - GlobalVariable::InternalLinkage, nullptr); - SmallVector<Constant *, 16> SizeVec; - for (unsigned i = 0; i < StructTy->getNumElements(); ++i) { - Type *Ty = StructTy->getElementType(i); - std::string Str; - raw_string_ostream StrOS(Str); - Ty->print(StrOS); - TypeNameVec.push_back( - ConstantExpr::getPointerCast( - createPrivateGlobalForString(M, StrOS.str(), true), - Int8PtrTy)); - OffsetVec.push_back( - ConstantInt::get(Int32Ty, - DL.getStructLayout(StructTy)->getElementOffset(i))); - SizeVec.push_back(ConstantInt::get(Int32Ty, - DL.getTypeAllocSize(Ty))); - } - TypeName->setInitializer(ConstantArray::get(TypeNameArrayTy, TypeNameVec)); - Offset->setInitializer(ConstantArray::get(OffsetArrayTy, OffsetVec)); - Size->setInitializer(ConstantArray::get(SizeArrayTy, SizeVec)); -} - -// Create the global variable for the cache-fragmentation tool. -GlobalVariable *EfficiencySanitizer::createCacheFragInfoGV( - Module &M, const DataLayout &DL, Constant *UnitName) { - assert(Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag); - - auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); - auto *Int8PtrPtrTy = Int8PtrTy->getPointerTo(); - auto *Int32Ty = Type::getInt32Ty(*Ctx); - auto *Int32PtrTy = Type::getInt32PtrTy(*Ctx); - auto *Int64Ty = Type::getInt64Ty(*Ctx); - auto *Int64PtrTy = Type::getInt64PtrTy(*Ctx); - // This structure should be kept consistent with the StructInfo struct - // in the runtime library. - // struct StructInfo { - // const char *StructName; - // u32 Size; - // u32 NumFields; - // u32 *FieldOffset; // auxiliary struct field info. - // u32 *FieldSize; // auxiliary struct field info. - // const char **FieldTypeName; // auxiliary struct field info. - // u64 *FieldCounters; - // u64 *ArrayCounter; - // }; - auto *StructInfoTy = - StructType::get(Int8PtrTy, Int32Ty, Int32Ty, Int32PtrTy, Int32PtrTy, - Int8PtrPtrTy, Int64PtrTy, Int64PtrTy); - auto *StructInfoPtrTy = StructInfoTy->getPointerTo(); - // This structure should be kept consistent with the CacheFragInfo struct - // in the runtime library. - // struct CacheFragInfo { - // const char *UnitName; - // u32 NumStructs; - // StructInfo *Structs; - // }; - auto *CacheFragInfoTy = StructType::get(Int8PtrTy, Int32Ty, StructInfoPtrTy); - - std::vector<StructType *> Vec = M.getIdentifiedStructTypes(); - unsigned NumStructs = 0; - SmallVector<Constant *, 16> Initializers; - - for (auto &StructTy : Vec) { - if (shouldIgnoreStructType(StructTy)) { - ++NumIgnoredStructs; - continue; - } - ++NumStructs; - - // StructName. - SmallString<MaxStructCounterNameSize> CounterNameStr; - createStructCounterName(StructTy, CounterNameStr); - GlobalVariable *StructCounterName = createPrivateGlobalForString( - M, CounterNameStr, /*AllowMerging*/true); - - // Counters. - // We create the counter array with StructCounterName and weak linkage - // so that the structs with the same name and layout from different - // compilation units will be merged into one. - auto *CounterArrayTy = ArrayType::get(Int64Ty, - getStructCounterSize(StructTy)); - GlobalVariable *Counters = - new GlobalVariable(M, CounterArrayTy, false, - GlobalVariable::WeakAnyLinkage, - ConstantAggregateZero::get(CounterArrayTy), - CounterNameStr); - - // Remember the counter variable for each struct type. - StructTyMap.insert(std::pair<Type *, GlobalVariable *>(StructTy, Counters)); - - // We pass the field type name array, offset array, and size array to - // the runtime for better reporting. - GlobalVariable *TypeName = nullptr, *Offset = nullptr, *Size = nullptr; - if (ClAuxFieldInfo) - createCacheFragAuxGV(M, DL, StructTy, TypeName, Offset, Size); - - Constant *FieldCounterIdx[2]; - FieldCounterIdx[0] = ConstantInt::get(Int32Ty, 0); - FieldCounterIdx[1] = ConstantInt::get(Int32Ty, - getFieldCounterIdx(StructTy)); - Constant *ArrayCounterIdx[2]; - ArrayCounterIdx[0] = ConstantInt::get(Int32Ty, 0); - ArrayCounterIdx[1] = ConstantInt::get(Int32Ty, - getArrayCounterIdx(StructTy)); - Initializers.push_back(ConstantStruct::get( - StructInfoTy, - ConstantExpr::getPointerCast(StructCounterName, Int8PtrTy), - ConstantInt::get(Int32Ty, - DL.getStructLayout(StructTy)->getSizeInBytes()), - ConstantInt::get(Int32Ty, StructTy->getNumElements()), - Offset == nullptr ? ConstantPointerNull::get(Int32PtrTy) - : ConstantExpr::getPointerCast(Offset, Int32PtrTy), - Size == nullptr ? ConstantPointerNull::get(Int32PtrTy) - : ConstantExpr::getPointerCast(Size, Int32PtrTy), - TypeName == nullptr - ? ConstantPointerNull::get(Int8PtrPtrTy) - : ConstantExpr::getPointerCast(TypeName, Int8PtrPtrTy), - ConstantExpr::getGetElementPtr(CounterArrayTy, Counters, - FieldCounterIdx), - ConstantExpr::getGetElementPtr(CounterArrayTy, Counters, - ArrayCounterIdx))); - } - // Structs. - Constant *StructInfo; - if (NumStructs == 0) { - StructInfo = ConstantPointerNull::get(StructInfoPtrTy); - } else { - auto *StructInfoArrayTy = ArrayType::get(StructInfoTy, NumStructs); - StructInfo = ConstantExpr::getPointerCast( - new GlobalVariable(M, StructInfoArrayTy, false, - GlobalVariable::InternalLinkage, - ConstantArray::get(StructInfoArrayTy, Initializers)), - StructInfoPtrTy); - } - - auto *CacheFragInfoGV = new GlobalVariable( - M, CacheFragInfoTy, true, GlobalVariable::InternalLinkage, - ConstantStruct::get(CacheFragInfoTy, UnitName, - ConstantInt::get(Int32Ty, NumStructs), StructInfo)); - return CacheFragInfoGV; -} - -// Create the tool-specific argument passed to EsanInit and EsanExit. -Constant *EfficiencySanitizer::createEsanInitToolInfoArg(Module &M, - const DataLayout &DL) { - // This structure contains tool-specific information about each compilation - // unit (module) and is passed to the runtime library. - GlobalVariable *ToolInfoGV = nullptr; - - auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); - // Compilation unit name. - auto *UnitName = ConstantExpr::getPointerCast( - createPrivateGlobalForString(M, M.getModuleIdentifier(), true), - Int8PtrTy); - - // Create the tool-specific variable. - if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag) - ToolInfoGV = createCacheFragInfoGV(M, DL, UnitName); - - if (ToolInfoGV != nullptr) - return ConstantExpr::getPointerCast(ToolInfoGV, Int8PtrTy); - - // Create the null pointer if no tool-specific variable created. - return ConstantPointerNull::get(Int8PtrTy); -} - -void EfficiencySanitizer::createDestructor(Module &M, Constant *ToolInfoArg) { - PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx); - EsanDtorFunction = Function::Create(FunctionType::get(Type::getVoidTy(*Ctx), - false), - GlobalValue::InternalLinkage, - EsanModuleDtorName, &M); - ReturnInst::Create(*Ctx, BasicBlock::Create(*Ctx, "", EsanDtorFunction)); - IRBuilder<> IRB_Dtor(EsanDtorFunction->getEntryBlock().getTerminator()); - Function *EsanExit = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(EsanExitName, IRB_Dtor.getVoidTy(), - Int8PtrTy)); - EsanExit->setLinkage(Function::ExternalLinkage); - IRB_Dtor.CreateCall(EsanExit, {ToolInfoArg}); - appendToGlobalDtors(M, EsanDtorFunction, EsanCtorAndDtorPriority); -} - -bool EfficiencySanitizer::initOnModule(Module &M) { - - Triple TargetTriple(M.getTargetTriple()); - if (TargetTriple.isMIPS64()) - ShadowParams = ShadowParams40; - else - ShadowParams = ShadowParams47; - - Ctx = &M.getContext(); - const DataLayout &DL = M.getDataLayout(); - IRBuilder<> IRB(M.getContext()); - IntegerType *OrdTy = IRB.getInt32Ty(); - PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx); - IntptrTy = DL.getIntPtrType(M.getContext()); - // Create the variable passed to EsanInit and EsanExit. - Constant *ToolInfoArg = createEsanInitToolInfoArg(M, DL); - // Constructor - // We specify the tool type both in the EsanWhichToolName global - // and as an arg to the init routine as a sanity check. - std::tie(EsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( - M, EsanModuleCtorName, EsanInitName, /*InitArgTypes=*/{OrdTy, Int8PtrTy}, - /*InitArgs=*/{ - ConstantInt::get(OrdTy, static_cast<int>(Options.ToolType)), - ToolInfoArg}); - appendToGlobalCtors(M, EsanCtorFunction, EsanCtorAndDtorPriority); - - createDestructor(M, ToolInfoArg); - - new GlobalVariable(M, OrdTy, true, - GlobalValue::WeakAnyLinkage, - ConstantInt::get(OrdTy, - static_cast<int>(Options.ToolType)), - EsanWhichToolName); - - return true; -} - -Value *EfficiencySanitizer::appToShadow(Value *Shadow, IRBuilder<> &IRB) { - // Shadow = ((App & Mask) + Offs) >> Scale - Shadow = IRB.CreateAnd(Shadow, ConstantInt::get(IntptrTy, ShadowParams.ShadowMask)); - uint64_t Offs; - int Scale = ShadowScale[Options.ToolType]; - if (Scale <= 2) - Offs = ShadowParams.ShadowOffs[Scale]; - else - Offs = ShadowParams.ShadowOffs[0] << Scale; - Shadow = IRB.CreateAdd(Shadow, ConstantInt::get(IntptrTy, Offs)); - if (Scale > 0) - Shadow = IRB.CreateLShr(Shadow, Scale); - return Shadow; -} - -bool EfficiencySanitizer::shouldIgnoreMemoryAccess(Instruction *I) { - if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag) { - // We'd like to know about cache fragmentation in vtable accesses and - // constant data references, so we do not currently ignore anything. - return false; - } else if (Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet) { - // TODO: the instrumentation disturbs the data layout on the stack, so we - // may want to add an option to ignore stack references (if we can - // distinguish them) to reduce overhead. - } - // TODO(bruening): future tools will be returning true for some cases. - return false; -} - -bool EfficiencySanitizer::runOnModule(Module &M) { - bool Res = initOnModule(M); - initializeCallbacks(M); - for (auto &F : M) { - Res |= runOnFunction(F, M); - } - return Res; -} - -bool EfficiencySanitizer::runOnFunction(Function &F, Module &M) { - // This is required to prevent instrumenting the call to __esan_init from - // within the module constructor. - if (&F == EsanCtorFunction) - return false; - SmallVector<Instruction *, 8> LoadsAndStores; - SmallVector<Instruction *, 8> MemIntrinCalls; - SmallVector<Instruction *, 8> GetElementPtrs; - bool Res = false; - const DataLayout &DL = M.getDataLayout(); - const TargetLibraryInfo *TLI = - &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); - - for (auto &BB : F) { - for (auto &Inst : BB) { - if ((isa<LoadInst>(Inst) || isa<StoreInst>(Inst) || - isa<AtomicRMWInst>(Inst) || isa<AtomicCmpXchgInst>(Inst)) && - !shouldIgnoreMemoryAccess(&Inst)) - LoadsAndStores.push_back(&Inst); - else if (isa<MemIntrinsic>(Inst)) - MemIntrinCalls.push_back(&Inst); - else if (isa<GetElementPtrInst>(Inst)) - GetElementPtrs.push_back(&Inst); - else if (CallInst *CI = dyn_cast<CallInst>(&Inst)) - maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI); - } - } - - if (ClInstrumentLoadsAndStores) { - for (auto Inst : LoadsAndStores) { - Res |= instrumentLoadOrStore(Inst, DL); - } - } - - if (ClInstrumentMemIntrinsics) { - for (auto Inst : MemIntrinCalls) { - Res |= instrumentMemIntrinsic(cast<MemIntrinsic>(Inst)); - } - } - - if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag) { - for (auto Inst : GetElementPtrs) { - Res |= instrumentGetElementPtr(Inst, M); - } - } - - return Res; -} - -bool EfficiencySanitizer::instrumentLoadOrStore(Instruction *I, - const DataLayout &DL) { - IRBuilder<> IRB(I); - bool IsStore; - Value *Addr; - unsigned Alignment; - if (LoadInst *Load = dyn_cast<LoadInst>(I)) { - IsStore = false; - Alignment = Load->getAlignment(); - Addr = Load->getPointerOperand(); - } else if (StoreInst *Store = dyn_cast<StoreInst>(I)) { - IsStore = true; - Alignment = Store->getAlignment(); - Addr = Store->getPointerOperand(); - } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) { - IsStore = true; - Alignment = 0; - Addr = RMW->getPointerOperand(); - } else if (AtomicCmpXchgInst *Xchg = dyn_cast<AtomicCmpXchgInst>(I)) { - IsStore = true; - Alignment = 0; - Addr = Xchg->getPointerOperand(); - } else - llvm_unreachable("Unsupported mem access type"); - - Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); - const uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8; - Value *OnAccessFunc = nullptr; - - // Convert 0 to the default alignment. - if (Alignment == 0) - Alignment = DL.getPrefTypeAlignment(OrigTy); - - if (IsStore) - NumInstrumentedStores++; - else - NumInstrumentedLoads++; - int Idx = getMemoryAccessFuncIndex(Addr, DL); - if (Idx < 0) { - OnAccessFunc = IsStore ? EsanUnalignedStoreN : EsanUnalignedLoadN; - IRB.CreateCall(OnAccessFunc, - {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), - ConstantInt::get(IntptrTy, TypeSizeBytes)}); - } else { - if (ClInstrumentFastpath && - instrumentFastpath(I, DL, IsStore, Addr, Alignment)) { - NumFastpaths++; - return true; - } - if (Alignment == 0 || (Alignment % TypeSizeBytes) == 0) - OnAccessFunc = IsStore ? EsanAlignedStore[Idx] : EsanAlignedLoad[Idx]; - else - OnAccessFunc = IsStore ? EsanUnalignedStore[Idx] : EsanUnalignedLoad[Idx]; - IRB.CreateCall(OnAccessFunc, - IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); - } - return true; -} - -// It's simplest to replace the memset/memmove/memcpy intrinsics with -// calls that the runtime library intercepts. -// Our pass is late enough that calls should not turn back into intrinsics. -bool EfficiencySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { - IRBuilder<> IRB(MI); - bool Res = false; - if (isa<MemSetInst>(MI)) { - IRB.CreateCall( - MemsetFn, - {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()), - IRB.CreateIntCast(MI->getArgOperand(1), IRB.getInt32Ty(), false), - IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)}); - MI->eraseFromParent(); - Res = true; - } else if (isa<MemTransferInst>(MI)) { - IRB.CreateCall( - isa<MemCpyInst>(MI) ? MemcpyFn : MemmoveFn, - {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()), - IRB.CreatePointerCast(MI->getArgOperand(1), IRB.getInt8PtrTy()), - IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)}); - MI->eraseFromParent(); - Res = true; - } else - llvm_unreachable("Unsupported mem intrinsic type"); - return Res; -} - -bool EfficiencySanitizer::instrumentGetElementPtr(Instruction *I, Module &M) { - GetElementPtrInst *GepInst = dyn_cast<GetElementPtrInst>(I); - bool Res = false; - if (GepInst == nullptr || GepInst->getNumIndices() == 1) { - ++NumIgnoredGEPs; - return false; - } - Type *SourceTy = GepInst->getSourceElementType(); - StructType *StructTy = nullptr; - ConstantInt *Idx; - // Check if GEP calculates address from a struct array. - if (isa<StructType>(SourceTy)) { - StructTy = cast<StructType>(SourceTy); - Idx = dyn_cast<ConstantInt>(GepInst->getOperand(1)); - if ((Idx == nullptr || Idx->getSExtValue() != 0) && - !shouldIgnoreStructType(StructTy) && StructTyMap.count(StructTy) != 0) - Res |= insertCounterUpdate(I, StructTy, getArrayCounterIdx(StructTy)); - } - // Iterate all (except the first and the last) idx within each GEP instruction - // for possible nested struct field address calculation. - for (unsigned i = 1; i < GepInst->getNumIndices(); ++i) { - SmallVector<Value *, 8> IdxVec(GepInst->idx_begin(), - GepInst->idx_begin() + i); - Type *Ty = GetElementPtrInst::getIndexedType(SourceTy, IdxVec); - unsigned CounterIdx = 0; - if (isa<ArrayType>(Ty)) { - ArrayType *ArrayTy = cast<ArrayType>(Ty); - StructTy = dyn_cast<StructType>(ArrayTy->getElementType()); - if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0) - continue; - // The last counter for struct array access. - CounterIdx = getArrayCounterIdx(StructTy); - } else if (isa<StructType>(Ty)) { - StructTy = cast<StructType>(Ty); - if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0) - continue; - // Get the StructTy's subfield index. - Idx = cast<ConstantInt>(GepInst->getOperand(i+1)); - assert(Idx->getSExtValue() >= 0 && - Idx->getSExtValue() < StructTy->getNumElements()); - CounterIdx = getFieldCounterIdx(StructTy) + Idx->getSExtValue(); - } - Res |= insertCounterUpdate(I, StructTy, CounterIdx); - } - if (Res) - ++NumInstrumentedGEPs; - else - ++NumIgnoredGEPs; - return Res; -} - -bool EfficiencySanitizer::insertCounterUpdate(Instruction *I, - StructType *StructTy, - unsigned CounterIdx) { - GlobalVariable *CounterArray = StructTyMap[StructTy]; - if (CounterArray == nullptr) - return false; - IRBuilder<> IRB(I); - Constant *Indices[2]; - // Xref http://llvm.org/docs/LangRef.html#i-getelementptr and - // http://llvm.org/docs/GetElementPtr.html. - // The first index of the GEP instruction steps through the first operand, - // i.e., the array itself. - Indices[0] = ConstantInt::get(IRB.getInt32Ty(), 0); - // The second index is the index within the array. - Indices[1] = ConstantInt::get(IRB.getInt32Ty(), CounterIdx); - Constant *Counter = - ConstantExpr::getGetElementPtr( - ArrayType::get(IRB.getInt64Ty(), getStructCounterSize(StructTy)), - CounterArray, Indices); - Value *Load = IRB.CreateLoad(Counter); - IRB.CreateStore(IRB.CreateAdd(Load, ConstantInt::get(IRB.getInt64Ty(), 1)), - Counter); - return true; -} - -int EfficiencySanitizer::getMemoryAccessFuncIndex(Value *Addr, - const DataLayout &DL) { - Type *OrigPtrTy = Addr->getType(); - Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); - assert(OrigTy->isSized()); - // The size is always a multiple of 8. - uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8; - if (TypeSizeBytes != 1 && TypeSizeBytes != 2 && TypeSizeBytes != 4 && - TypeSizeBytes != 8 && TypeSizeBytes != 16) { - // Irregular sizes do not have per-size call targets. - NumAccessesWithIrregularSize++; - return -1; - } - size_t Idx = countTrailingZeros(TypeSizeBytes); - assert(Idx < NumberOfAccessSizes); - return Idx; -} - -bool EfficiencySanitizer::instrumentFastpath(Instruction *I, - const DataLayout &DL, bool IsStore, - Value *Addr, unsigned Alignment) { - if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag) { - return instrumentFastpathCacheFrag(I, DL, Addr, Alignment); - } else if (Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet) { - return instrumentFastpathWorkingSet(I, DL, Addr, Alignment); - } - return false; -} - -bool EfficiencySanitizer::instrumentFastpathCacheFrag(Instruction *I, - const DataLayout &DL, - Value *Addr, - unsigned Alignment) { - // Do nothing. - return true; // Return true to avoid slowpath instrumentation. -} - -bool EfficiencySanitizer::instrumentFastpathWorkingSet( - Instruction *I, const DataLayout &DL, Value *Addr, unsigned Alignment) { - assert(ShadowScale[Options.ToolType] == 6); // The code below assumes this - IRBuilder<> IRB(I); - Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); - const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); - // Bail to the slowpath if the access might touch multiple cache lines. - // An access aligned to its size is guaranteed to be intra-cache-line. - // getMemoryAccessFuncIndex has already ruled out a size larger than 16 - // and thus larger than a cache line for platforms this tool targets - // (and our shadow memory setup assumes 64-byte cache lines). - assert(TypeSize <= 128); - if (!(TypeSize == 8 || - (Alignment % (TypeSize / 8)) == 0)) { - if (ClAssumeIntraCacheLine) - ++NumAssumedIntraCacheLine; - else - return false; - } - - // We inline instrumentation to set the corresponding shadow bits for - // each cache line touched by the application. Here we handle a single - // load or store where we've already ruled out the possibility that it - // might touch more than one cache line and thus we simply update the - // shadow memory for a single cache line. - // Our shadow memory model is fine with races when manipulating shadow values. - // We generate the following code: - // - // const char BitMask = 0x81; - // char *ShadowAddr = appToShadow(AppAddr); - // if ((*ShadowAddr & BitMask) != BitMask) - // *ShadowAddr |= Bitmask; - // - Value *AddrPtr = IRB.CreatePointerCast(Addr, IntptrTy); - Value *ShadowPtr = appToShadow(AddrPtr, IRB); - Type *ShadowTy = IntegerType::get(*Ctx, 8U); - Type *ShadowPtrTy = PointerType::get(ShadowTy, 0); - // The bottom bit is used for the current sampling period's working set. - // The top bit is used for the total working set. We set both on each - // memory access, if they are not already set. - Value *ValueMask = ConstantInt::get(ShadowTy, 0x81); // 10000001B - - Value *OldValue = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); - // The AND and CMP will be turned into a TEST instruction by the compiler. - Value *Cmp = IRB.CreateICmpNE(IRB.CreateAnd(OldValue, ValueMask), ValueMask); - Instruction *CmpTerm = SplitBlockAndInsertIfThen(Cmp, I, false); - // FIXME: do I need to call SetCurrentDebugLocation? - IRB.SetInsertPoint(CmpTerm); - // We use OR to set the shadow bits to avoid corrupting the middle 6 bits, - // which are used by the runtime library. - Value *NewVal = IRB.CreateOr(OldValue, ValueMask); - IRB.CreateStore(NewVal, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); - IRB.SetInsertPoint(I); - - return true; -} diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp index 9af64ed332cdb..59950ffc4e9a6 100644 --- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -1,9 +1,8 @@ //===- GCOVProfiling.cpp - Insert edge counters for gcov profiling --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -103,11 +102,11 @@ private: std::vector<Regex> &Regexes); // Get pointers to the functions in the runtime library. - Constant *getStartFileFunc(); - Constant *getEmitFunctionFunc(); - Constant *getEmitArcsFunc(); - Constant *getSummaryInfoFunc(); - Constant *getEndFileFunc(); + FunctionCallee getStartFileFunc(); + FunctionCallee getEmitFunctionFunc(); + FunctionCallee getEmitArcsFunc(); + FunctionCallee getSummaryInfoFunc(); + FunctionCallee getEndFileFunc(); // Add the function to write out all our counters to the global destructor // list. @@ -648,7 +647,7 @@ void GCOVProfiler::AddFlushBeforeForkAndExec() { for (auto I : ForkAndExecs) { IRBuilder<> Builder(I); FunctionType *FTy = FunctionType::get(Builder.getVoidTy(), {}, false); - Constant *GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy); + FunctionCallee GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy); Builder.CreateCall(GCOVFlush); I->getParent()->splitBasicBlock(I); } @@ -811,14 +810,14 @@ bool GCOVProfiler::emitProfileArcs() { auto It = EdgeToCounter.find({Pred, &BB}); assert(It != EdgeToCounter.end()); const unsigned Edge = It->second; - Value *EdgeCounter = - BuilderForPhi.CreateConstInBoundsGEP2_64(Counters, 0, Edge); + Value *EdgeCounter = BuilderForPhi.CreateConstInBoundsGEP2_64( + Counters->getValueType(), Counters, 0, Edge); Phi->addIncoming(EdgeCounter, Pred); } // Skip phis, landingpads. IRBuilder<> Builder(&*BB.getFirstInsertionPt()); - Value *Count = Builder.CreateLoad(Phi); + Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), Phi); Count = Builder.CreateAdd(Count, Builder.getInt64(1)); Builder.CreateStore(Count, Phi); @@ -827,9 +826,9 @@ bool GCOVProfiler::emitProfileArcs() { auto It = EdgeToCounter.find({&BB, nullptr}); assert(It != EdgeToCounter.end()); const unsigned Edge = It->second; - Value *Counter = - Builder.CreateConstInBoundsGEP2_64(Counters, 0, Edge); - Value *Count = Builder.CreateLoad(Counter); + Value *Counter = Builder.CreateConstInBoundsGEP2_64( + Counters->getValueType(), Counters, 0, Edge); + Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), Counter); Count = Builder.CreateAdd(Count, Builder.getInt64(1)); Builder.CreateStore(Count, Counter); } @@ -864,7 +863,7 @@ bool GCOVProfiler::emitProfileArcs() { // Initialize the environment and register the local writeout and flush // functions. - Constant *GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy); + FunctionCallee GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy); Builder.CreateCall(GCOVInit, {WriteoutF, FlushF}); Builder.CreateRetVoid(); @@ -874,22 +873,21 @@ bool GCOVProfiler::emitProfileArcs() { return Result; } -Constant *GCOVProfiler::getStartFileFunc() { +FunctionCallee GCOVProfiler::getStartFileFunc() { Type *Args[] = { Type::getInt8PtrTy(*Ctx), // const char *orig_filename Type::getInt8PtrTy(*Ctx), // const char version[4] Type::getInt32Ty(*Ctx), // uint32_t checksum }; FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false); - auto *Res = M->getOrInsertFunction("llvm_gcda_start_file", FTy); - if (Function *FunRes = dyn_cast<Function>(Res)) - if (auto AK = TLI->getExtAttrForI32Param(false)) - FunRes->addParamAttr(2, AK); + AttributeList AL; + if (auto AK = TLI->getExtAttrForI32Param(false)) + AL = AL.addParamAttribute(*Ctx, 2, AK); + FunctionCallee Res = M->getOrInsertFunction("llvm_gcda_start_file", FTy, AL); return Res; - } -Constant *GCOVProfiler::getEmitFunctionFunc() { +FunctionCallee GCOVProfiler::getEmitFunctionFunc() { Type *Args[] = { Type::getInt32Ty(*Ctx), // uint32_t ident Type::getInt8PtrTy(*Ctx), // const char *function_name @@ -898,36 +896,34 @@ Constant *GCOVProfiler::getEmitFunctionFunc() { Type::getInt32Ty(*Ctx), // uint32_t cfg_checksum }; FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false); - auto *Res = M->getOrInsertFunction("llvm_gcda_emit_function", FTy); - if (Function *FunRes = dyn_cast<Function>(Res)) - if (auto AK = TLI->getExtAttrForI32Param(false)) { - FunRes->addParamAttr(0, AK); - FunRes->addParamAttr(2, AK); - FunRes->addParamAttr(3, AK); - FunRes->addParamAttr(4, AK); - } - return Res; + AttributeList AL; + if (auto AK = TLI->getExtAttrForI32Param(false)) { + AL = AL.addParamAttribute(*Ctx, 0, AK); + AL = AL.addParamAttribute(*Ctx, 2, AK); + AL = AL.addParamAttribute(*Ctx, 3, AK); + AL = AL.addParamAttribute(*Ctx, 4, AK); + } + return M->getOrInsertFunction("llvm_gcda_emit_function", FTy); } -Constant *GCOVProfiler::getEmitArcsFunc() { +FunctionCallee GCOVProfiler::getEmitArcsFunc() { Type *Args[] = { Type::getInt32Ty(*Ctx), // uint32_t num_counters Type::getInt64PtrTy(*Ctx), // uint64_t *counters }; FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false); - auto *Res = M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy); - if (Function *FunRes = dyn_cast<Function>(Res)) - if (auto AK = TLI->getExtAttrForI32Param(false)) - FunRes->addParamAttr(0, AK); - return Res; + AttributeList AL; + if (auto AK = TLI->getExtAttrForI32Param(false)) + AL = AL.addParamAttribute(*Ctx, 0, AK); + return M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy, AL); } -Constant *GCOVProfiler::getSummaryInfoFunc() { +FunctionCallee GCOVProfiler::getSummaryInfoFunc() { FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false); return M->getOrInsertFunction("llvm_gcda_summary_info", FTy); } -Constant *GCOVProfiler::getEndFileFunc() { +FunctionCallee GCOVProfiler::getEndFileFunc() { FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false); return M->getOrInsertFunction("llvm_gcda_end_file", FTy); } @@ -947,11 +943,11 @@ Function *GCOVProfiler::insertCounterWriteout( BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", WriteoutF); IRBuilder<> Builder(BB); - Constant *StartFile = getStartFileFunc(); - Constant *EmitFunction = getEmitFunctionFunc(); - Constant *EmitArcs = getEmitArcsFunc(); - Constant *SummaryInfo = getSummaryInfoFunc(); - Constant *EndFile = getEndFileFunc(); + FunctionCallee StartFile = getStartFileFunc(); + FunctionCallee EmitFunction = getEmitFunctionFunc(); + FunctionCallee EmitArcs = getEmitArcsFunc(); + FunctionCallee SummaryInfo = getSummaryInfoFunc(); + FunctionCallee EndFile = getEndFileFunc(); NamedMDNode *CUNodes = M->getNamedMetadata("llvm.dbg.cu"); if (!CUNodes) { @@ -1088,22 +1084,32 @@ Function *GCOVProfiler::insertCounterWriteout( PHINode *IV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2); IV->addIncoming(Builder.getInt32(0), BB); - auto *FileInfoPtr = - Builder.CreateInBoundsGEP(FileInfoArrayGV, {Builder.getInt32(0), IV}); - auto *StartFileCallArgsPtr = Builder.CreateStructGEP(FileInfoPtr, 0); + auto *FileInfoPtr = Builder.CreateInBoundsGEP( + FileInfoArrayTy, FileInfoArrayGV, {Builder.getInt32(0), IV}); + auto *StartFileCallArgsPtr = + Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0); auto *StartFileCall = Builder.CreateCall( StartFile, - {Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 0)), - Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 1)), - Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 2))}); + {Builder.CreateLoad(StartFileCallArgsTy->getElementType(0), + Builder.CreateStructGEP(StartFileCallArgsTy, + StartFileCallArgsPtr, 0)), + Builder.CreateLoad(StartFileCallArgsTy->getElementType(1), + Builder.CreateStructGEP(StartFileCallArgsTy, + StartFileCallArgsPtr, 1)), + Builder.CreateLoad(StartFileCallArgsTy->getElementType(2), + Builder.CreateStructGEP(StartFileCallArgsTy, + StartFileCallArgsPtr, 2))}); if (auto AK = TLI->getExtAttrForI32Param(false)) StartFileCall->addParamAttr(2, AK); auto *NumCounters = - Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 1)); + Builder.CreateLoad(FileInfoTy->getElementType(1), + Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1)); auto *EmitFunctionCallArgsArray = - Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 2)); + Builder.CreateLoad(FileInfoTy->getElementType(2), + Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2)); auto *EmitArcsCallArgsArray = - Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 3)); + Builder.CreateLoad(FileInfoTy->getElementType(3), + Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3)); auto *EnterCounterLoopCond = Builder.CreateICmpSLT(Builder.getInt32(0), NumCounters); Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch); @@ -1111,16 +1117,26 @@ Function *GCOVProfiler::insertCounterWriteout( Builder.SetInsertPoint(CounterLoopHeader); auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2); JV->addIncoming(Builder.getInt32(0), FileLoopHeader); - auto *EmitFunctionCallArgsPtr = - Builder.CreateInBoundsGEP(EmitFunctionCallArgsArray, {JV}); + auto *EmitFunctionCallArgsPtr = Builder.CreateInBoundsGEP( + EmitFunctionCallArgsTy, EmitFunctionCallArgsArray, JV); auto *EmitFunctionCall = Builder.CreateCall( EmitFunction, - {Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 0)), - Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 1)), - Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 2)), - Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 3)), - Builder.CreateLoad( - Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 4))}); + {Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(0), + Builder.CreateStructGEP(EmitFunctionCallArgsTy, + EmitFunctionCallArgsPtr, 0)), + Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(1), + Builder.CreateStructGEP(EmitFunctionCallArgsTy, + EmitFunctionCallArgsPtr, 1)), + Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(2), + Builder.CreateStructGEP(EmitFunctionCallArgsTy, + EmitFunctionCallArgsPtr, 2)), + Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(3), + Builder.CreateStructGEP(EmitFunctionCallArgsTy, + EmitFunctionCallArgsPtr, 3)), + Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(4), + Builder.CreateStructGEP(EmitFunctionCallArgsTy, + EmitFunctionCallArgsPtr, + 4))}); if (auto AK = TLI->getExtAttrForI32Param(false)) { EmitFunctionCall->addParamAttr(0, AK); EmitFunctionCall->addParamAttr(2, AK); @@ -1128,11 +1144,15 @@ Function *GCOVProfiler::insertCounterWriteout( EmitFunctionCall->addParamAttr(4, AK); } auto *EmitArcsCallArgsPtr = - Builder.CreateInBoundsGEP(EmitArcsCallArgsArray, {JV}); + Builder.CreateInBoundsGEP(EmitArcsCallArgsTy, EmitArcsCallArgsArray, JV); auto *EmitArcsCall = Builder.CreateCall( EmitArcs, - {Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 0)), - Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 1))}); + {Builder.CreateLoad( + EmitArcsCallArgsTy->getElementType(0), + Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0)), + Builder.CreateLoad(EmitArcsCallArgsTy->getElementType(1), + Builder.CreateStructGEP(EmitArcsCallArgsTy, + EmitArcsCallArgsPtr, 1))}); if (auto AK = TLI->getExtAttrForI32Param(false)) EmitArcsCall->addParamAttr(0, AK); auto *NextJV = Builder.CreateAdd(JV, Builder.getInt32(1)); @@ -1172,7 +1192,7 @@ insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) { BasicBlock *Entry = BasicBlock::Create(*Ctx, "entry", FlushF); // Write out the current counters. - Constant *WriteoutF = M->getFunction("__llvm_gcov_writeout"); + Function *WriteoutF = M->getFunction("__llvm_gcov_writeout"); assert(WriteoutF && "Need to create the writeout function first!"); IRBuilder<> Builder(Entry); diff --git a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index d04c2b76288f4..90a9f4955a4b4 100644 --- a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -1,9 +1,8 @@ //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,6 +11,7 @@ /// based on tagged addressing. //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" @@ -21,6 +21,7 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" @@ -125,10 +126,10 @@ static cl::opt<bool> ClEnableKhwasan( // is accessed. The shadow mapping looks like: // Shadow = (Mem >> scale) + offset -static cl::opt<unsigned long long> ClMappingOffset( - "hwasan-mapping-offset", - cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden, - cl::init(0)); +static cl::opt<uint64_t> + ClMappingOffset("hwasan-mapping-offset", + cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), + cl::Hidden, cl::init(0)); static cl::opt<bool> ClWithIfunc("hwasan-with-ifunc", @@ -148,42 +149,46 @@ static cl::opt<bool> "in a thread-local ring buffer"), cl::Hidden, cl::init(true)); static cl::opt<bool> - ClCreateFrameDescriptions("hwasan-create-frame-descriptions", - cl::desc("create static frame descriptions"), - cl::Hidden, cl::init(true)); - -static cl::opt<bool> ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics", cl::desc("instrument memory intrinsics"), cl::Hidden, cl::init(true)); + +static cl::opt<bool> + ClInstrumentLandingPads("hwasan-instrument-landing-pads", + cl::desc("instrument landing pads"), cl::Hidden, + cl::init(true)); + +static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks", + cl::desc("inline all checks"), + cl::Hidden, cl::init(false)); + namespace { /// An instrumentation pass implementing detection of addressability bugs /// using tagged pointers. -class HWAddressSanitizer : public FunctionPass { +class HWAddressSanitizer { public: - // Pass identification, replacement for typeid. - static char ID; - - explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false) - : FunctionPass(ID) { + explicit HWAddressSanitizer(Module &M, bool CompileKernel = false, + bool Recover = false) { this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ? ClEnableKhwasan : CompileKernel; - } - StringRef getPassName() const override { return "HWAddressSanitizer"; } + initializeModule(M); + } - bool runOnFunction(Function &F) override; - bool doInitialization(Module &M) override; + bool sanitizeFunction(Function &F); + void initializeModule(Module &M); void initializeCallbacks(Module &M); + Value *getDynamicShadowIfunc(IRBuilder<> &IRB); Value *getDynamicShadowNonTls(IRBuilder<> &IRB); void untagPointerOperand(Instruction *I, Value *Addr); - Value *memToShadow(Value *Shadow, Type *Ty, IRBuilder<> &IRB); - void instrumentMemAccessInline(Value *PtrLong, bool IsWrite, + Value *shadowBase(); + Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); + void instrumentMemAccessInline(Value *Ptr, bool IsWrite, unsigned AccessSizeIndex, Instruction *InsertBefore); void instrumentMemIntrinsic(MemIntrinsic *MI); @@ -193,11 +198,15 @@ public: Value **MaybeMask); bool isInterestingAlloca(const AllocaInst &AI); - bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag); + bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size); Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag); Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong); - bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas, - SmallVectorImpl<Instruction *> &RetVec, Value *StackTag); + bool instrumentStack( + SmallVectorImpl<AllocaInst *> &Allocas, + DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap, + SmallVectorImpl<Instruction *> &RetVec, Value *StackTag); + Value *readRegister(IRBuilder<> &IRB, StringRef Name); + bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec); Value *getNextTagWithCall(IRBuilder<> &IRB); Value *getStackBaseTag(IRBuilder<> &IRB); Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI, @@ -205,31 +214,14 @@ public: Value *getUARTag(IRBuilder<> &IRB, Value *StackTag); Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); - Value *emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord); + void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord); private: LLVMContext *C; std::string CurModuleUniqueId; Triple TargetTriple; - Function *HWAsanMemmove, *HWAsanMemcpy, *HWAsanMemset; - - // Frame description is a way to pass names/sizes of local variables - // to the run-time w/o adding extra executable code in every function. - // We do this by creating a separate section with {PC,Descr} pairs and passing - // the section beg/end to __hwasan_init_frames() at module init time. - std::string createFrameString(ArrayRef<AllocaInst*> Allocas); - void createFrameGlobal(Function &F, const std::string &FrameString); - // Get the section name for frame descriptions. Currently ELF-only. - const char *getFrameSection() { return "__hwasan_frames"; } - const char *getFrameSectionBeg() { return "__start___hwasan_frames"; } - const char *getFrameSectionEnd() { return "__stop___hwasan_frames"; } - GlobalVariable *createFrameSectionBound(Module &M, Type *Ty, - const char *Name) { - auto GV = new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, - nullptr, Name); - GV->setVisibility(GlobalValue::HiddenVisibility); - return GV; - } + FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset; + FunctionCallee HWAsanHandleVfork; /// This struct defines the shadow mapping using the rule: /// shadow = (mem >> Scale) + Offset. @@ -253,48 +245,95 @@ private: Type *IntptrTy; Type *Int8PtrTy; Type *Int8Ty; + Type *Int32Ty; bool CompileKernel; bool Recover; Function *HwasanCtorFunction; - Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes]; - Function *HwasanMemoryAccessCallbackSized[2]; + FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes]; + FunctionCallee HwasanMemoryAccessCallbackSized[2]; - Function *HwasanTagMemoryFunc; - Function *HwasanGenerateTagFunc; - Function *HwasanThreadEnterFunc; + FunctionCallee HwasanTagMemoryFunc; + FunctionCallee HwasanGenerateTagFunc; + FunctionCallee HwasanThreadEnterFunc; Constant *ShadowGlobal; Value *LocalDynamicShadow = nullptr; + Value *StackBaseTag = nullptr; GlobalValue *ThreadPtrGlobal = nullptr; }; +class HWAddressSanitizerLegacyPass : public FunctionPass { +public: + // Pass identification, replacement for typeid. + static char ID; + + explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false, + bool Recover = false) + : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {} + + StringRef getPassName() const override { return "HWAddressSanitizer"; } + + bool doInitialization(Module &M) override { + HWASan = llvm::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover); + return true; + } + + bool runOnFunction(Function &F) override { + return HWASan->sanitizeFunction(F); + } + + bool doFinalization(Module &M) override { + HWASan.reset(); + return false; + } + +private: + std::unique_ptr<HWAddressSanitizer> HWASan; + bool CompileKernel; + bool Recover; +}; + } // end anonymous namespace -char HWAddressSanitizer::ID = 0; +char HWAddressSanitizerLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN( - HWAddressSanitizer, "hwasan", + HWAddressSanitizerLegacyPass, "hwasan", "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false) INITIALIZE_PASS_END( - HWAddressSanitizer, "hwasan", + HWAddressSanitizerLegacyPass, "hwasan", "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false) -FunctionPass *llvm::createHWAddressSanitizerPass(bool CompileKernel, - bool Recover) { +FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel, + bool Recover) { assert(!CompileKernel || Recover); - return new HWAddressSanitizer(CompileKernel, Recover); + return new HWAddressSanitizerLegacyPass(CompileKernel, Recover); +} + +HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover) + : CompileKernel(CompileKernel), Recover(Recover) {} + +PreservedAnalyses HWAddressSanitizerPass::run(Module &M, + ModuleAnalysisManager &MAM) { + HWAddressSanitizer HWASan(M, CompileKernel, Recover); + bool Modified = false; + for (Function &F : M) + Modified |= HWASan.sanitizeFunction(F); + if (Modified) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); } /// Module-level initialization. /// /// inserts a call to __hwasan_init to the module's constructor list. -bool HWAddressSanitizer::doInitialization(Module &M) { +void HWAddressSanitizer::initializeModule(Module &M) { LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n"); auto &DL = M.getDataLayout(); @@ -308,47 +347,35 @@ bool HWAddressSanitizer::doInitialization(Module &M) { IntptrTy = IRB.getIntPtrTy(DL); Int8PtrTy = IRB.getInt8PtrTy(); Int8Ty = IRB.getInt8Ty(); + Int32Ty = IRB.getInt32Ty(); HwasanCtorFunction = nullptr; if (!CompileKernel) { std::tie(HwasanCtorFunction, std::ignore) = - createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName, - kHwasanInitName, - /*InitArgTypes=*/{}, - /*InitArgs=*/{}); - Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName); - HwasanCtorFunction->setComdat(CtorComdat); - appendToGlobalCtors(M, HwasanCtorFunction, 0, HwasanCtorFunction); - - // Create a zero-length global in __hwasan_frame so that the linker will - // always create start and stop symbols. - // - // N.B. If we ever start creating associated metadata in this pass this - // global will need to be associated with the ctor. - Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0); - auto GV = - new GlobalVariable(M, Int8Arr0Ty, /*isConstantGlobal*/ true, - GlobalVariable::PrivateLinkage, - Constant::getNullValue(Int8Arr0Ty), "__hwasan"); - GV->setSection(getFrameSection()); - GV->setComdat(CtorComdat); - appendToCompilerUsed(M, GV); - - IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator()); - IRBCtor.CreateCall( - declareSanitizerInitFunction(M, "__hwasan_init_frames", - {Int8PtrTy, Int8PtrTy}), - {createFrameSectionBound(M, Int8Ty, getFrameSectionBeg()), - createFrameSectionBound(M, Int8Ty, getFrameSectionEnd())}); + getOrCreateSanitizerCtorAndInitFunctions( + M, kHwasanModuleCtorName, kHwasanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + // This callback is invoked when the functions are created the first + // time. Hook them into the global ctors list in that case: + [&](Function *Ctor, FunctionCallee) { + Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName); + Ctor->setComdat(CtorComdat); + appendToGlobalCtors(M, Ctor, 0, Ctor); + }); } - if (!TargetTriple.isAndroid()) - appendToCompilerUsed( - M, ThreadPtrGlobal = new GlobalVariable( - M, IntptrTy, false, GlobalVariable::ExternalLinkage, nullptr, - "__hwasan_tls", nullptr, GlobalVariable::InitialExecTLSModel)); - - return true; + if (!TargetTriple.isAndroid()) { + Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] { + auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false, + GlobalValue::ExternalLinkage, nullptr, + "__hwasan_tls", nullptr, + GlobalVariable::InitialExecTLSModel); + appendToCompilerUsed(M, GV); + return GV; + }); + ThreadPtrGlobal = cast<GlobalVariable>(C); + } } void HWAddressSanitizer::initializeCallbacks(Module &M) { @@ -357,44 +384,55 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) { const std::string TypeStr = AccessIsWrite ? "store" : "load"; const std::string EndingStr = Recover ? "_noabort" : ""; - HwasanMemoryAccessCallbackSized[AccessIsWrite] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, - FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false))); + HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction( + ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, + FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)); for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; AccessSizeIndex++) { HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( + M.getOrInsertFunction( ClMemoryAccessCallbackPrefix + TypeStr + itostr(1ULL << AccessSizeIndex) + EndingStr, - FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false))); + FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)); } } - HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy)); - HwasanGenerateTagFunc = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty)); + HwasanTagMemoryFunc = M.getOrInsertFunction( + "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy); + HwasanGenerateTagFunc = + M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty); - if (Mapping.InGlobal) - ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow", - ArrayType::get(IRB.getInt8Ty(), 0)); + ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow", + ArrayType::get(IRB.getInt8Ty(), 0)); const std::string MemIntrinCallbackPrefix = CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix; - HWAsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); - HWAsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy)); - HWAsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy)); + HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IntptrTy); + HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IntptrTy); + HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset", + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), + IRB.getInt32Ty(), IntptrTy); - HwasanThreadEnterFunc = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy())); + HWAsanHandleVfork = + M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy); + + HwasanThreadEnterFunc = + M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy()); +} + +Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) { + // An empty inline asm with input reg == output reg. + // An opaque no-op cast, basically. + InlineAsm *Asm = InlineAsm::get( + FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false), + StringRef(""), StringRef("=r,0"), + /*hasSideEffects=*/false); + return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow"); } Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) { @@ -403,18 +441,12 @@ Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) { return nullptr; if (Mapping.InGlobal) { - // An empty inline asm with input reg == output reg. - // An opaque pointer-to-int cast, basically. - InlineAsm *Asm = InlineAsm::get( - FunctionType::get(IntptrTy, {ShadowGlobal->getType()}, false), - StringRef(""), StringRef("=r,0"), - /*hasSideEffects=*/false); - return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow"); + return getDynamicShadowIfunc(IRB); } else { Value *GlobalDynamicAddress = IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal( - kHwasanShadowMemoryDynamicAddress, IntptrTy); - return IRB.CreateLoad(GlobalDynamicAddress); + kHwasanShadowMemoryDynamicAddress, Int8PtrTy); + return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress); } } @@ -506,29 +538,44 @@ void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) { I->setOperand(getPointerOperandIndex(I), UntaggedPtr); } -Value *HWAddressSanitizer::memToShadow(Value *Mem, Type *Ty, IRBuilder<> &IRB) { +Value *HWAddressSanitizer::shadowBase() { + if (LocalDynamicShadow) + return LocalDynamicShadow; + return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset), + Int8PtrTy); +} + +Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) { // Mem >> Scale Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale); if (Mapping.Offset == 0) - return Shadow; + return IRB.CreateIntToPtr(Shadow, Int8PtrTy); // (Mem >> Scale) + Offset - Value *ShadowBase; - if (LocalDynamicShadow) - ShadowBase = LocalDynamicShadow; - else - ShadowBase = ConstantInt::get(Ty, Mapping.Offset); - return IRB.CreateAdd(Shadow, ShadowBase); + return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow); } -void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite, +void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite, unsigned AccessSizeIndex, Instruction *InsertBefore) { + const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex; IRBuilder<> IRB(InsertBefore); + + if (!ClInlineAllChecks && TargetTriple.isAArch64() && + TargetTriple.isOSBinFormatELF() && !Recover) { + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy); + IRB.CreateCall( + Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess), + {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)}); + return; + } + + Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy); Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty()); Value *AddrLong = untagPointer(IRB, PtrLong); - Value *ShadowLong = memToShadow(AddrLong, PtrLong->getType(), IRB); - Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, Int8PtrTy)); + Value *Shadow = memToShadow(AddrLong, IRB); + Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow); Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag); int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ? @@ -540,11 +587,35 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite, } Instruction *CheckTerm = - SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover, + SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false, MDBuilder(*C).createBranchWeights(1, 100000)); IRB.SetInsertPoint(CheckTerm); - const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex; + Value *OutOfShortGranuleTagRange = + IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15)); + Instruction *CheckFailTerm = + SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover, + MDBuilder(*C).createBranchWeights(1, 100000)); + + IRB.SetInsertPoint(CheckTerm); + Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty); + PtrLowBits = IRB.CreateAdd( + PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1)); + Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag); + SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false, + MDBuilder(*C).createBranchWeights(1, 100000), + nullptr, nullptr, CheckFailTerm->getParent()); + + IRB.SetInsertPoint(CheckTerm); + Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15); + InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy); + Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr); + Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag); + SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false, + MDBuilder(*C).createBranchWeights(1, 100000), + nullptr, nullptr, CheckFailTerm->getParent()); + + IRB.SetInsertPoint(CheckFailTerm); InlineAsm *Asm; switch (TargetTriple.getArch()) { case Triple::x86_64: @@ -568,6 +639,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite, report_fatal_error("unsupported architecture"); } IRB.CreateCall(Asm, PtrLong); + if (Recover) + cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent()); } void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { @@ -610,7 +683,6 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) { return false; //FIXME IRBuilder<> IRB(I); - Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); if (isPowerOf2_64(TypeSize) && (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) && (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 || @@ -618,13 +690,14 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) { size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize); if (ClInstrumentWithCalls) { IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex], - AddrLong); + IRB.CreatePointerCast(Addr, IntptrTy)); } else { - instrumentMemAccessInline(AddrLong, IsWrite, AccessSizeIndex, I); + instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I); } } else { IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite], - {AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8)}); + {IRB.CreatePointerCast(Addr, IntptrTy), + ConstantInt::get(IntptrTy, TypeSize / 8)}); } untagPointerOperand(I, Addr); @@ -644,27 +717,33 @@ static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) { } bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, - Value *Tag) { - size_t Size = (getAllocaSizeInBytes(*AI) + Mapping.getAllocaAlignment() - 1) & - ~(Mapping.getAllocaAlignment() - 1); + Value *Tag, size_t Size) { + size_t AlignedSize = alignTo(Size, Mapping.getAllocaAlignment()); Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty()); if (ClInstrumentWithCalls) { IRB.CreateCall(HwasanTagMemoryFunc, {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag, - ConstantInt::get(IntptrTy, Size)}); + ConstantInt::get(IntptrTy, AlignedSize)}); } else { size_t ShadowSize = Size >> Mapping.Scale; - Value *ShadowPtr = IRB.CreateIntToPtr( - memToShadow(IRB.CreatePointerCast(AI, IntptrTy), AI->getType(), IRB), - Int8PtrTy); + Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB); // If this memset is not inlined, it will be intercepted in the hwasan // runtime library. That's OK, because the interceptor skips the checks if // the address is in the shadow region. // FIXME: the interceptor is not as fast as real memset. Consider lowering // llvm.memset right here into either a sequence of stores, or a call to // hwasan_tag_memory. - IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1); + if (ShadowSize) + IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1); + if (Size != AlignedSize) { + IRB.CreateStore( + ConstantInt::get(Int8Ty, Size % Mapping.getAllocaAlignment()), + IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize)); + IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32( + Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy), + AlignedSize - 1)); + } } return true; } @@ -674,10 +753,16 @@ static unsigned RetagMask(unsigned AllocaNo) { // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these // masks. // The list does not include the value 255, which is used for UAR. - static unsigned FastMasks[] = { - 0, 1, 2, 3, 4, 6, 7, 8, 12, 14, 15, 16, 24, - 28, 30, 31, 32, 48, 56, 60, 62, 63, 64, 96, 112, 120, - 124, 126, 127, 128, 192, 224, 240, 248, 252, 254}; + // + // Because we are more likely to use earlier elements of this list than later + // ones, it is sorted in increasing order of probability of collision with a + // mask allocated (temporally) nearby. The program that generated this list + // can be found at: + // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py + static unsigned FastMasks[] = {0, 128, 64, 192, 32, 96, 224, 112, 240, + 48, 16, 120, 248, 56, 24, 8, 124, 252, + 60, 28, 12, 4, 126, 254, 62, 30, 14, + 6, 2, 127, 63, 31, 15, 7, 3, 1}; return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))]; } @@ -688,6 +773,8 @@ Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) { Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) { if (ClGenerateTagsWithCalls) return getNextTagWithCall(IRB); + if (StackBaseTag) + return StackBaseTag; // FIXME: use addressofreturnaddress (but implement it in aarch64 backend // first). Module *M = IRB.GetInsertBlock()->getParent()->getParent(); @@ -763,7 +850,8 @@ Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { Function *ThreadPointerFunc = Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); Value *SlotPtr = IRB.CreatePointerCast( - IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), 0x30), + IRB.CreateConstGEP1_32(IRB.getInt8Ty(), + IRB.CreateCall(ThreadPointerFunc), 0x30), Ty->getPointerTo(0)); return SlotPtr; } @@ -774,45 +862,21 @@ Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { return nullptr; } -// Creates a string with a description of the stack frame (set of Allocas). -// The string is intended to be human readable. -// The current form is: Size1 Name1; Size2 Name2; ... -std::string -HWAddressSanitizer::createFrameString(ArrayRef<AllocaInst *> Allocas) { - std::ostringstream Descr; - for (auto AI : Allocas) - Descr << getAllocaSizeInBytes(*AI) << " " << AI->getName().str() << "; "; - return Descr.str(); -} - -// Creates a global in the frame section which consists of two pointers: -// the function PC and the frame string constant. -void HWAddressSanitizer::createFrameGlobal(Function &F, - const std::string &FrameString) { - Module &M = *F.getParent(); - auto DescrGV = createPrivateGlobalForString(M, FrameString, true); - auto PtrPairTy = StructType::get(F.getType(), DescrGV->getType()); - auto GV = new GlobalVariable( - M, PtrPairTy, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage, - ConstantStruct::get(PtrPairTy, (Constant *)&F, (Constant *)DescrGV), - "__hwasan"); - GV->setSection(getFrameSection()); - appendToCompilerUsed(M, GV); - // Put GV into the F's Comadat so that if F is deleted GV can be deleted too. - if (auto Comdat = - GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId)) - GV->setComdat(Comdat); -} +void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { + if (!Mapping.InTls) { + LocalDynamicShadow = getDynamicShadowNonTls(IRB); + return; + } -Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, - bool WithFrameRecord) { - if (!Mapping.InTls) - return getDynamicShadowNonTls(IRB); + if (!WithFrameRecord && TargetTriple.isAndroid()) { + LocalDynamicShadow = getDynamicShadowIfunc(IRB); + return; + } Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); assert(SlotPtr); - Instruction *ThreadLong = IRB.CreateLoad(SlotPtr); + Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); Function *F = IRB.GetInsertBlock()->getParent(); if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") { @@ -826,7 +890,7 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, // FIXME: This should call a new runtime function with a custom calling // convention to avoid needing to spill all arguments here. IRB.CreateCall(HwasanThreadEnterFunc); - LoadInst *ReloadThreadLong = IRB.CreateLoad(SlotPtr); + LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin()); PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2); @@ -840,15 +904,21 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); if (WithFrameRecord) { + StackBaseTag = IRB.CreateAShr(ThreadLong, 3); + // Prepare ring buffer data. - auto PC = IRB.CreatePtrToInt(F, IntptrTy); + Value *PC; + if (TargetTriple.getArch() == Triple::aarch64) + PC = readRegister(IRB, "pc"); + else + PC = IRB.CreatePtrToInt(F, IntptrTy); auto GetStackPointerFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::frameaddress); Value *SP = IRB.CreatePtrToInt( IRB.CreateCall(GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())}), IntptrTy); - // Mix SP and PC. TODO: also add the tag to the mix. + // Mix SP and PC. // Assumptions: // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero) // SP is 0xsssssssssssSSSS0 (4 lower bits are zero) @@ -879,16 +949,38 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, // Get shadow base address by aligning RecordPtr up. // Note: this is not correct if the pointer is already aligned. // Runtime library will make sure this never happens. - Value *ShadowBase = IRB.CreateAdd( + LocalDynamicShadow = IRB.CreateAdd( IRB.CreateOr( ThreadLongMaybeUntagged, ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)), ConstantInt::get(IntptrTy, 1), "hwasan.shadow"); - return ShadowBase; + LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy); +} + +Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) { + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + Function *ReadRegister = + Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy); + MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)}); + Value *Args[] = {MetadataAsValue::get(*C, MD)}; + return IRB.CreateCall(ReadRegister, Args); +} + +bool HWAddressSanitizer::instrumentLandingPads( + SmallVectorImpl<Instruction *> &LandingPadVec) { + for (auto *LP : LandingPadVec) { + IRBuilder<> IRB(LP->getNextNode()); + IRB.CreateCall( + HWAsanHandleVfork, + {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp" + : "sp")}); + } + return true; } bool HWAddressSanitizer::instrumentStack( SmallVectorImpl<AllocaInst *> &Allocas, + DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap, SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) { // Ideally, we want to calculate tagged stack base pointer, and rewrite all // alloca addresses using that. Unfortunately, offsets are not known yet @@ -913,14 +1005,22 @@ bool HWAddressSanitizer::instrumentStack( U.set(Replacement); } - tagAlloca(IRB, AI, Tag); + for (auto *DDI : AllocaDeclareMap.lookup(AI)) { + DIExpression *OldExpr = DDI->getExpression(); + DIExpression *NewExpr = DIExpression::append( + OldExpr, {dwarf::DW_OP_LLVM_tag_offset, RetagMask(N)}); + DDI->setArgOperand(2, MetadataAsValue::get(*C, NewExpr)); + } + + size_t Size = getAllocaSizeInBytes(*AI); + tagAlloca(IRB, AI, Tag, Size); for (auto RI : RetVec) { IRB.SetInsertPoint(RI); // Re-tag alloca memory with the special UAR tag. Value *Tag = getUARTag(IRB, StackTag); - tagAlloca(IRB, AI, Tag); + tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getAllocaAlignment())); } } @@ -943,7 +1043,7 @@ bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) { !AI.isSwiftError()); } -bool HWAddressSanitizer::runOnFunction(Function &F) { +bool HWAddressSanitizer::sanitizeFunction(Function &F) { if (&F == HwasanCtorFunction) return false; @@ -955,15 +1055,12 @@ bool HWAddressSanitizer::runOnFunction(Function &F) { SmallVector<Instruction*, 16> ToInstrument; SmallVector<AllocaInst*, 8> AllocasToInstrument; SmallVector<Instruction*, 8> RetVec; + SmallVector<Instruction*, 8> LandingPadVec; + DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> AllocaDeclareMap; for (auto &BB : F) { for (auto &Inst : BB) { if (ClInstrumentStack) if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) { - // Realign all allocas. We don't want small uninteresting allocas to - // hide in instrumented alloca's padding. - if (AI->getAlignment() < Mapping.getAllocaAlignment()) - AI->setAlignment(Mapping.getAllocaAlignment()); - // Instrument some of them. if (isInterestingAlloca(*AI)) AllocasToInstrument.push_back(AI); continue; @@ -973,6 +1070,13 @@ bool HWAddressSanitizer::runOnFunction(Function &F) { isa<CleanupReturnInst>(Inst)) RetVec.push_back(&Inst); + if (auto *DDI = dyn_cast<DbgDeclareInst>(&Inst)) + if (auto *Alloca = dyn_cast_or_null<AllocaInst>(DDI->getAddress())) + AllocaDeclareMap[Alloca].push_back(DDI); + + if (ClInstrumentLandingPads && isa<LandingPadInst>(Inst)) + LandingPadVec.push_back(&Inst); + Value *MaybeMask = nullptr; bool IsWrite; unsigned Alignment; @@ -984,33 +1088,93 @@ bool HWAddressSanitizer::runOnFunction(Function &F) { } } - if (AllocasToInstrument.empty() && ToInstrument.empty()) - return false; + initializeCallbacks(*F.getParent()); - if (ClCreateFrameDescriptions && !AllocasToInstrument.empty()) - createFrameGlobal(F, createFrameString(AllocasToInstrument)); + if (!LandingPadVec.empty()) + instrumentLandingPads(LandingPadVec); - initializeCallbacks(*F.getParent()); + if (AllocasToInstrument.empty() && ToInstrument.empty()) + return false; assert(!LocalDynamicShadow); Instruction *InsertPt = &*F.getEntryBlock().begin(); IRBuilder<> EntryIRB(InsertPt); - LocalDynamicShadow = emitPrologue(EntryIRB, - /*WithFrameRecord*/ ClRecordStackHistory && - !AllocasToInstrument.empty()); + emitPrologue(EntryIRB, + /*WithFrameRecord*/ ClRecordStackHistory && + !AllocasToInstrument.empty()); bool Changed = false; if (!AllocasToInstrument.empty()) { Value *StackTag = ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB); - Changed |= instrumentStack(AllocasToInstrument, RetVec, StackTag); + Changed |= instrumentStack(AllocasToInstrument, AllocaDeclareMap, RetVec, + StackTag); + } + + // Pad and align each of the allocas that we instrumented to stop small + // uninteresting allocas from hiding in instrumented alloca's padding and so + // that we have enough space to store real tags for short granules. + DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap; + for (AllocaInst *AI : AllocasToInstrument) { + uint64_t Size = getAllocaSizeInBytes(*AI); + uint64_t AlignedSize = alignTo(Size, Mapping.getAllocaAlignment()); + AI->setAlignment(std::max(AI->getAlignment(), 16u)); + if (Size != AlignedSize) { + Type *AllocatedType = AI->getAllocatedType(); + if (AI->isArrayAllocation()) { + uint64_t ArraySize = + cast<ConstantInt>(AI->getArraySize())->getZExtValue(); + AllocatedType = ArrayType::get(AllocatedType, ArraySize); + } + Type *TypeWithPadding = StructType::get( + AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size)); + auto *NewAI = new AllocaInst( + TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI); + NewAI->takeName(AI); + NewAI->setAlignment(AI->getAlignment()); + NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca()); + NewAI->setSwiftError(AI->isSwiftError()); + NewAI->copyMetadata(*AI); + auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI); + AI->replaceAllUsesWith(Bitcast); + AllocaToPaddedAllocaMap[AI] = NewAI; + } + } + + if (!AllocaToPaddedAllocaMap.empty()) { + for (auto &BB : F) + for (auto &Inst : BB) + if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst)) + if (auto *AI = + dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation())) + if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI)) + DVI->setArgOperand( + 0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI))); + for (auto &P : AllocaToPaddedAllocaMap) + P.first->eraseFromParent(); + } + + // If we split the entry block, move any allocas that were originally in the + // entry block back into the entry block so that they aren't treated as + // dynamic allocas. + if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) { + InsertPt = &*F.getEntryBlock().begin(); + for (auto II = EntryIRB.GetInsertBlock()->begin(), + IE = EntryIRB.GetInsertBlock()->end(); + II != IE;) { + Instruction *I = &*II++; + if (auto *AI = dyn_cast<AllocaInst>(I)) + if (isa<ConstantInt>(AI->getArraySize())) + I->moveBefore(InsertPt); + } } for (auto Inst : ToInstrument) Changed |= instrumentMemAccess(Inst); LocalDynamicShadow = nullptr; + StackBaseTag = nullptr; return Changed; } diff --git a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 58436c8560ad5..c7371f567ff3a 100644 --- a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -1,9 +1,8 @@ //===- IndirectCallPromotion.cpp - Optimizations based on value profiling -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -239,7 +238,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite( LLVM_DEBUG(dbgs() << " Candidate " << I << " Count=" << Count << " Target_func: " << Target << "\n"); - if (ICPInvokeOnly && dyn_cast<CallInst>(Inst)) { + if (ICPInvokeOnly && isa<CallInst>(Inst)) { LLVM_DEBUG(dbgs() << " Not promote: User options.\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst) @@ -247,7 +246,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite( }); break; } - if (ICPCallOnly && dyn_cast<InvokeInst>(Inst)) { + if (ICPCallOnly && isa<InvokeInst>(Inst)) { LLVM_DEBUG(dbgs() << " Not promote: User option.\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst) @@ -311,10 +310,10 @@ Instruction *llvm::pgo::promoteIndirectCall(Instruction *Inst, promoteCallWithIfThenElse(CallSite(Inst), DirectCallee, BranchWeights); if (AttachProfToDirectCall) { - SmallVector<uint32_t, 1> Weights; - Weights.push_back(Count); MDBuilder MDB(NewInst->getContext()); - NewInst->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + NewInst->setMetadata( + LLVMContext::MD_prof, + MDB.createBranchWeights({static_cast<uint32_t>(Count)})); } using namespace ore; @@ -394,9 +393,7 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, } bool Changed = false; for (auto &F : M) { - if (F.isDeclaration()) - continue; - if (F.hasFnAttribute(Attribute::OptimizeNone)) + if (F.isDeclaration() || F.hasOptNone()) continue; std::unique_ptr<OptimizationRemarkEmitter> OwnedORE; diff --git a/lib/Transforms/Instrumentation/InstrOrderFile.cpp b/lib/Transforms/Instrumentation/InstrOrderFile.cpp new file mode 100644 index 0000000000000..a2c1ddfd279ea --- /dev/null +++ b/lib/Transforms/Instrumentation/InstrOrderFile.cpp @@ -0,0 +1,211 @@ +//===- InstrOrderFile.cpp ---- Late IR instrumentation for order file ----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/PassRegistry.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/InstrOrderFile.h" +#include <fstream> +#include <map> +#include <mutex> +#include <set> +#include <sstream> + +using namespace llvm; +#define DEBUG_TYPE "instrorderfile" + +static cl::opt<std::string> ClOrderFileWriteMapping( + "orderfile-write-mapping", cl::init(""), + cl::desc( + "Dump functions and their MD5 hash to deobfuscate profile data"), + cl::Hidden); + +namespace { + +// We need a global bitmap to tell if a function is executed. We also +// need a global variable to save the order of functions. We can use a +// fixed-size buffer that saves the MD5 hash of the function. We need +// a global variable to save the index into the buffer. + +std::mutex MappingMutex; + +struct InstrOrderFile { +private: + GlobalVariable *OrderFileBuffer; + GlobalVariable *BufferIdx; + GlobalVariable *BitMap; + ArrayType *BufferTy; + ArrayType *MapTy; + +public: + InstrOrderFile() {} + + void createOrderFileData(Module &M) { + LLVMContext &Ctx = M.getContext(); + int NumFunctions = 0; + for (Function &F : M) { + if (!F.isDeclaration()) + NumFunctions++; + } + + BufferTy = + ArrayType::get(Type::getInt64Ty(Ctx), INSTR_ORDER_FILE_BUFFER_SIZE); + Type *IdxTy = Type::getInt32Ty(Ctx); + MapTy = ArrayType::get(Type::getInt8Ty(Ctx), NumFunctions); + + // Create the global variables. + std::string SymbolName = INSTR_PROF_ORDERFILE_BUFFER_NAME_STR; + OrderFileBuffer = new GlobalVariable(M, BufferTy, false, GlobalValue::LinkOnceODRLinkage, + Constant::getNullValue(BufferTy), SymbolName); + Triple TT = Triple(M.getTargetTriple()); + OrderFileBuffer->setSection( + getInstrProfSectionName(IPSK_orderfile, TT.getObjectFormat())); + + std::string IndexName = INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR; + BufferIdx = new GlobalVariable(M, IdxTy, false, GlobalValue::LinkOnceODRLinkage, + Constant::getNullValue(IdxTy), IndexName); + + std::string BitMapName = "bitmap_0"; + BitMap = new GlobalVariable(M, MapTy, false, GlobalValue::PrivateLinkage, + Constant::getNullValue(MapTy), BitMapName); + } + + // Generate the code sequence in the entry block of each function to + // update the buffer. + void generateCodeSequence(Module &M, Function &F, int FuncId) { + if (!ClOrderFileWriteMapping.empty()) { + std::lock_guard<std::mutex> LogLock(MappingMutex); + std::error_code EC; + llvm::raw_fd_ostream OS(ClOrderFileWriteMapping, EC, llvm::sys::fs::F_Append); + if (EC) { + report_fatal_error(Twine("Failed to open ") + ClOrderFileWriteMapping + + " to save mapping file for order file instrumentation\n"); + } else { + std::stringstream stream; + stream << std::hex << MD5Hash(F.getName()); + std::string singleLine = "MD5 " + stream.str() + " " + + std::string(F.getName()) + '\n'; + OS << singleLine; + } + } + + BasicBlock *OrigEntry = &F.getEntryBlock(); + + LLVMContext &Ctx = M.getContext(); + IntegerType *Int32Ty = Type::getInt32Ty(Ctx); + IntegerType *Int8Ty = Type::getInt8Ty(Ctx); + + // Create a new entry block for instrumentation. We will check the bitmap + // in this basic block. + BasicBlock *NewEntry = + BasicBlock::Create(M.getContext(), "order_file_entry", &F, OrigEntry); + IRBuilder<> entryB(NewEntry); + // Create a basic block for updating the circular buffer. + BasicBlock *UpdateOrderFileBB = + BasicBlock::Create(M.getContext(), "order_file_set", &F, OrigEntry); + IRBuilder<> updateB(UpdateOrderFileBB); + + // Check the bitmap, if it is already 1, do nothing. + // Otherwise, set the bit, grab the index, update the buffer. + Value *IdxFlags[] = {ConstantInt::get(Int32Ty, 0), + ConstantInt::get(Int32Ty, FuncId)}; + Value *MapAddr = entryB.CreateGEP(MapTy, BitMap, IdxFlags, ""); + LoadInst *loadBitMap = entryB.CreateLoad(Int8Ty, MapAddr, ""); + entryB.CreateStore(ConstantInt::get(Int8Ty, 1), MapAddr); + Value *IsNotExecuted = + entryB.CreateICmpEQ(loadBitMap, ConstantInt::get(Int8Ty, 0)); + entryB.CreateCondBr(IsNotExecuted, UpdateOrderFileBB, OrigEntry); + + // Fill up UpdateOrderFileBB: grab the index, update the buffer! + Value *IdxVal = updateB.CreateAtomicRMW( + AtomicRMWInst::Add, BufferIdx, ConstantInt::get(Int32Ty, 1), + AtomicOrdering::SequentiallyConsistent); + // We need to wrap around the index to fit it inside the buffer. + Value *WrappedIdx = updateB.CreateAnd( + IdxVal, ConstantInt::get(Int32Ty, INSTR_ORDER_FILE_BUFFER_MASK)); + Value *BufferGEPIdx[] = {ConstantInt::get(Int32Ty, 0), WrappedIdx}; + Value *BufferAddr = + updateB.CreateGEP(BufferTy, OrderFileBuffer, BufferGEPIdx, ""); + updateB.CreateStore(ConstantInt::get(Type::getInt64Ty(Ctx), MD5Hash(F.getName())), + BufferAddr); + updateB.CreateBr(OrigEntry); + } + + bool run(Module &M) { + createOrderFileData(M); + + int FuncId = 0; + for (Function &F : M) { + if (F.isDeclaration()) + continue; + generateCodeSequence(M, F, FuncId); + ++FuncId; + } + + return true; + } + +}; // End of InstrOrderFile struct + +class InstrOrderFileLegacyPass : public ModulePass { +public: + static char ID; + + InstrOrderFileLegacyPass() : ModulePass(ID) { + initializeInstrOrderFileLegacyPassPass( + *PassRegistry::getPassRegistry()); + } + + bool runOnModule(Module &M) override; +}; + +} // End anonymous namespace + +bool InstrOrderFileLegacyPass::runOnModule(Module &M) { + if (skipModule(M)) + return false; + + return InstrOrderFile().run(M); +} + +PreservedAnalyses +InstrOrderFilePass::run(Module &M, ModuleAnalysisManager &AM) { + if (InstrOrderFile().run(M)) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +INITIALIZE_PASS_BEGIN(InstrOrderFileLegacyPass, "instrorderfile", + "Instrumentation for Order File", false, false) +INITIALIZE_PASS_END(InstrOrderFileLegacyPass, "instrorderfile", + "Instrumentation for Order File", false, false) + +char InstrOrderFileLegacyPass::ID = 0; + +ModulePass *llvm::createInstrOrderFilePass() { + return new InstrOrderFileLegacyPass(); +} diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp index 15b94388cbe51..63c2b8078967a 100644 --- a/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -1,9 +1,8 @@ //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Attributes.h" @@ -148,8 +149,8 @@ public: static char ID; InstrProfilingLegacyPass() : ModulePass(ID) {} - InstrProfilingLegacyPass(const InstrProfOptions &Options) - : ModulePass(ID), InstrProf(Options) {} + InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false) + : ModulePass(ID), InstrProf(Options, IsCS) {} StringRef getPassName() const override { return "Frontend instrumentation-based coverage lowering"; @@ -187,7 +188,7 @@ public: SSA.AddAvailableValue(PH, Init); } - void doExtraRewritesBeforeFinalDeletion() const override { + void doExtraRewritesBeforeFinalDeletion() override { for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { BasicBlock *ExitBlock = ExitBlocks[i]; Instruction *InsertPos = InsertPts[i]; @@ -196,6 +197,7 @@ public: // block. Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); Value *Addr = cast<StoreInst>(Store)->getPointerOperand(); + Type *Ty = LiveInValue->getType(); IRBuilder<> Builder(InsertPos); if (AtomicCounterUpdatePromoted) // automic update currently can only be promoted across the current @@ -203,7 +205,7 @@ public: Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, AtomicOrdering::SequentiallyConsistent); else { - LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted"); + LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted"); auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue); auto *NewStore = Builder.CreateStore(NewVal, Addr); @@ -232,9 +234,9 @@ class PGOCounterPromoter { public: PGOCounterPromoter( DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, - Loop &CurLoop, LoopInfo &LI) + Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI) : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop), - LI(LI) { + LI(LI), BFI(BFI) { SmallVector<BasicBlock *, 8> LoopExitBlocks; SmallPtrSet<BasicBlock *, 8> BlockSet; @@ -263,6 +265,20 @@ public: SSAUpdater SSA(&NewPHIs); Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); + // If BFI is set, we will use it to guide the promotions. + if (BFI) { + auto *BB = Cand.first->getParent(); + auto InstrCount = BFI->getBlockProfileCount(BB); + if (!InstrCount) + continue; + auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader()); + // If the average loop trip count is not greater than 1.5, we skip + // promotion. + if (PreheaderCount && + (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2)) + continue; + } + PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(), ExitBlocks, InsertPts, LoopToCandidates, LI); @@ -312,6 +328,11 @@ private: SmallVector<BasicBlock *, 8> ExitingBlocks; LP->getExitingBlocks(ExitingBlocks); + + // If BFI is set, we do more aggressive promotions based on BFI. + if (BFI) + return (unsigned)-1; + // Not considierered speculative. if (ExitingBlocks.size() == 1) return MaxNumOfPromotionsPerLoop; @@ -343,6 +364,7 @@ private: SmallVector<Instruction *, 8> InsertPts; Loop &L; LoopInfo &LI; + BlockFrequencyInfo *BFI; }; } // end anonymous namespace @@ -365,8 +387,9 @@ INITIALIZE_PASS_END( "Frontend instrumentation-based coverage lowering.", false, false) ModulePass * -llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) { - return new InstrProfilingLegacyPass(Options); +llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options, + bool IsCS) { + return new InstrProfilingLegacyPass(Options, IsCS); } static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) { @@ -415,6 +438,13 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) { LoopInfo LI(DT); DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates; + std::unique_ptr<BlockFrequencyInfo> BFI; + if (Options.UseBFIInPromotion) { + std::unique_ptr<BranchProbabilityInfo> BPI; + BPI.reset(new BranchProbabilityInfo(*F, LI, TLI)); + BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI)); + } + for (const auto &LoadStore : PromotionCandidates) { auto *CounterLoad = LoadStore.first; auto *CounterStore = LoadStore.second; @@ -430,7 +460,7 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) { // Do a post-order traversal of the loops so that counter updates can be // iteratively hoisted outside the loop nest. for (auto *Loop : llvm::reverse(Loops)) { - PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI); + PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get()); Promoter.run(&TotalCountersPromoted); } } @@ -509,13 +539,16 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) { return true; } -static Constant *getOrInsertValueProfilingCall(Module &M, - const TargetLibraryInfo &TLI, - bool IsRange = false) { +static FunctionCallee +getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, + bool IsRange = false) { LLVMContext &Ctx = M.getContext(); auto *ReturnTy = Type::getVoidTy(M.getContext()); - Constant *Res; + AttributeList AL; + if (auto AK = TLI.getExtAttrForI32Param(false)) + AL = AL.addParamAttribute(M.getContext(), 2, AK); + if (!IsRange) { Type *ParamTypes[] = { #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType @@ -523,8 +556,8 @@ static Constant *getOrInsertValueProfilingCall(Module &M, }; auto *ValueProfilingCallTy = FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); - Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(), - ValueProfilingCallTy); + return M.getOrInsertFunction(getInstrProfValueProfFuncName(), + ValueProfilingCallTy, AL); } else { Type *RangeParamTypes[] = { #define VALUE_RANGE_PROF 1 @@ -534,15 +567,9 @@ static Constant *getOrInsertValueProfilingCall(Module &M, }; auto *ValueRangeProfilingCallTy = FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false); - Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(), - ValueRangeProfilingCallTy); - } - - if (Function *FunRes = dyn_cast<Function>(Res)) { - if (auto AK = TLI.getExtAttrForI32Param(false)) - FunRes->addParamAttr(2, AK); + return M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(), + ValueRangeProfilingCallTy, AL); } - return Res; } void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { @@ -601,13 +628,15 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { IRBuilder<> Builder(Inc); uint64_t Index = Inc->getIndex()->getZExtValue(); - Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index); + Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(), + Counters, 0, Index); if (Options.Atomic || AtomicCounterUpdateAll) { Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), AtomicOrdering::Monotonic); } else { - Value *Load = Builder.CreateLoad(Addr, "pgocount"); + Value *IncStep = Inc->getStep(); + Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount"); auto *Count = Builder.CreateAdd(Load, Inc->getStep()); auto *Store = Builder.CreateStore(Count, Addr); if (isCounterPromotionEnabled()) @@ -678,32 +707,14 @@ static inline bool shouldRecordFunctionAddr(Function *F) { return F->hasAddressTaken() || F->hasLinkOnceLinkage(); } -static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F, - InstrProfIncrementInst *Inc) { - if (!needsComdatForCounter(F, M)) - return nullptr; - - // COFF format requires a COMDAT section to have a key symbol with the same - // name. The linker targeting COFF also requires that the COMDAT - // a section is associated to must precede the associating section. For this - // reason, we must choose the counter var's name as the name of the comdat. - StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF() - ? getInstrProfCountersVarPrefix() - : getInstrProfComdatPrefix()); - return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix))); -} - -static bool needsRuntimeRegistrationOfSectionRange(const Module &M) { +static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { // Don't do this for Darwin. compiler-rt uses linker magic. - if (Triple(M.getTargetTriple()).isOSDarwin()) + if (TT.isOSDarwin()) return false; - // Use linker script magic to get data/cnts/name start/end. - if (Triple(M.getTargetTriple()).isOSLinux() || - Triple(M.getTargetTriple()).isOSFreeBSD() || - Triple(M.getTargetTriple()).isOSNetBSD() || - Triple(M.getTargetTriple()).isOSFuchsia() || - Triple(M.getTargetTriple()).isPS4CPU()) + if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() || + TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() || + TT.isOSWindows()) return false; return true; @@ -720,13 +731,37 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { PD = It->second; } - // Move the name variable to the right section. Place them in a COMDAT group - // if the associated function is a COMDAT. This will make sure that - // only one copy of counters of the COMDAT function will be emitted after - // linking. + // Match the linkage and visibility of the name global, except on COFF, where + // the linkage must be local and consequentially the visibility must be + // default. Function *Fn = Inc->getParent()->getParent(); - Comdat *ProfileVarsComdat = nullptr; - ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc); + GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); + GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); + if (TT.isOSBinFormatCOFF()) { + Linkage = GlobalValue::InternalLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + // Move the name variable to the right section. Place them in a COMDAT group + // if the associated function is a COMDAT. This will make sure that only one + // copy of counters of the COMDAT function will be emitted after linking. Keep + // in mind that this pass may run before the inliner, so we need to create a + // new comdat group for the counters and profiling data. If we use the comdat + // of the parent function, that will result in relocations against discarded + // sections. + Comdat *Cmdt = nullptr; + GlobalValue::LinkageTypes CounterLinkage = Linkage; + if (needsComdatForCounter(*Fn, *M)) { + StringRef CmdtPrefix = getInstrProfComdatPrefix(); + if (TT.isOSBinFormatCOFF()) { + // For COFF, the comdat group name must be the name of a symbol in the + // group. Use the counter variable name, and upgrade its linkage to + // something externally visible, like linkonce_odr. + CmdtPrefix = getInstrProfCountersVarPrefix(); + CounterLinkage = GlobalValue::LinkOnceODRLinkage; + } + Cmdt = M->getOrInsertComdat(getVarName(Inc, CmdtPrefix)); + } uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); LLVMContext &Ctx = M->getContext(); @@ -734,20 +769,21 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { // Create the counters variable. auto *CounterPtr = - new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(), + new GlobalVariable(*M, CounterTy, false, Linkage, Constant::getNullValue(CounterTy), getVarName(Inc, getInstrProfCountersVarPrefix())); - CounterPtr->setVisibility(NamePtr->getVisibility()); + CounterPtr->setVisibility(Visibility); CounterPtr->setSection( getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); CounterPtr->setAlignment(8); - CounterPtr->setComdat(ProfileVarsComdat); + CounterPtr->setComdat(Cmdt); + CounterPtr->setLinkage(CounterLinkage); auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for // the current function. Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy); - if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) { + if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) { uint64_t NS = 0; for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) NS += PD.NumValueSites[Kind]; @@ -755,14 +791,14 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); auto *ValuesVar = - new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(), + new GlobalVariable(*M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), getVarName(Inc, getInstrProfValuesVarPrefix())); - ValuesVar->setVisibility(NamePtr->getVisibility()); + ValuesVar->setVisibility(Visibility); ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(8); - ValuesVar->setComdat(ProfileVarsComdat); + ValuesVar->setComdat(Cmdt); ValuesPtrExpr = ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } @@ -789,13 +825,13 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, #include "llvm/ProfileData/InstrProfData.inc" }; - auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(), + auto *Data = new GlobalVariable(*M, DataTy, false, Linkage, ConstantStruct::get(DataTy, DataVals), getVarName(Inc, getInstrProfDataVarPrefix())); - Data->setVisibility(NamePtr->getVisibility()); + Data->setVisibility(Visibility); Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT); - Data->setComdat(ProfileVarsComdat); + Data->setComdat(Cmdt); PD.RegionCounters = CounterPtr; PD.DataVar = Data; @@ -820,7 +856,7 @@ void InstrProfiling::emitVNodes() { // For now only support this on platforms that do // not require runtime registration to discover // named section start/end. - if (needsRuntimeRegistrationOfSectionRange(*M)) + if (needsRuntimeRegistrationOfSectionRange(TT)) return; size_t TotalNS = 0; @@ -881,6 +917,10 @@ void InstrProfiling::emitNameData() { NamesSize = CompressedNameStr.size(); NamesVar->setSection( getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); + // On COFF, it's important to reduce the alignment down to 1 to prevent the + // linker from inserting padding before the start of the names section or + // between names entries. + NamesVar->setAlignment(1); UsedVars.push_back(NamesVar); for (auto *NamePtr : ReferencedNames) @@ -888,7 +928,7 @@ void InstrProfiling::emitNameData() { } void InstrProfiling::emitRegistration() { - if (!needsRuntimeRegistrationOfSectionRange(*M)) + if (!needsRuntimeRegistrationOfSectionRange(TT)) return; // Construct the function. @@ -929,7 +969,7 @@ void InstrProfiling::emitRegistration() { bool InstrProfiling::emitRuntimeHook() { // We expect the linker to be invoked with -u<hook_var> flag for linux, // for which case there is no need to emit the user function. - if (Triple(M->getTargetTriple()).isOSLinux()) + if (TT.isOSLinux()) return false; // If the module's provided its own runtime, we don't need to do anything. @@ -950,11 +990,11 @@ bool InstrProfiling::emitRuntimeHook() { if (Options.NoRedZone) User->addFnAttr(Attribute::NoRedZone); User->setVisibility(GlobalValue::HiddenVisibility); - if (Triple(M->getTargetTriple()).supportsCOMDAT()) + if (TT.supportsCOMDAT()) User->setComdat(M->getOrInsertComdat(User->getName())); IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); - auto *Load = IRB.CreateLoad(Var); + auto *Load = IRB.CreateLoad(Int32Ty, Var); IRB.CreateRet(Load); // Mark the user variable as used so that it isn't stripped out. @@ -968,23 +1008,13 @@ void InstrProfiling::emitUses() { } void InstrProfiling::emitInitialization() { - StringRef InstrProfileOutput = Options.InstrProfileOutput; - - if (!InstrProfileOutput.empty()) { - // Create variable for profile name. - Constant *ProfileNameConst = - ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true); - GlobalVariable *ProfileNameVar = new GlobalVariable( - *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, - ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)); - if (TT.supportsCOMDAT()) { - ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); - ProfileNameVar->setComdat(M->getOrInsertComdat( - StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)))); - } - } - - Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName()); + // Create ProfileFileName variable. Don't don't this for the + // context-sensitive instrumentation lowering: This lowering is after + // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should + // have already create the variable before LTO/ThinLTO linking. + if (!IsCS) + createProfileFileNameVar(*M, Options.InstrProfileOutput); + Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); if (!RegisterF) return; @@ -1000,8 +1030,7 @@ void InstrProfiling::emitInitialization() { // Add the basic block and the necessary calls. IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); - if (RegisterF) - IRB.CreateCall(RegisterF, {}); + IRB.CreateCall(RegisterF, {}); IRB.CreateRetVoid(); appendToGlobalCtors(*M, F, 0); diff --git a/lib/Transforms/Instrumentation/Instrumentation.cpp b/lib/Transforms/Instrumentation/Instrumentation.cpp index c3e323613c707..f56a1bd91b898 100644 --- a/lib/Transforms/Instrumentation/Instrumentation.cpp +++ b/lib/Transforms/Instrumentation/Instrumentation.cpp @@ -1,9 +1,8 @@ //===-- Instrumentation.cpp - TransformUtils Infrastructure ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,10 +24,12 @@ using namespace llvm; /// Moves I before IP. Returns new insert point. static BasicBlock::iterator moveBeforeInsertPoint(BasicBlock::iterator I, BasicBlock::iterator IP) { // If I is IP, move the insert point down. - if (I == IP) - return ++IP; - // Otherwise, move I before IP and return IP. - I->moveBefore(&*IP); + if (I == IP) { + ++IP; + } else { + // Otherwise, move I before IP and return IP. + I->moveBefore(&*IP); + } return IP; } @@ -101,8 +102,8 @@ Comdat *llvm::GetOrCreateFunctionComdat(Function &F, Triple &T, /// initializeInstrumentation - Initialize all passes in the TransformUtils /// library. void llvm::initializeInstrumentation(PassRegistry &Registry) { - initializeAddressSanitizerPass(Registry); - initializeAddressSanitizerModulePass(Registry); + initializeAddressSanitizerLegacyPassPass(Registry); + initializeModuleAddressSanitizerLegacyPassPass(Registry); initializeBoundsCheckingLegacyPassPass(Registry); initializeControlHeightReductionLegacyPassPass(Registry); initializeGCOVProfilerLegacyPassPass(Registry); @@ -110,13 +111,13 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) { initializePGOInstrumentationUseLegacyPassPass(Registry); initializePGOIndirectCallPromotionLegacyPassPass(Registry); initializePGOMemOPSizeOptLegacyPassPass(Registry); + initializeInstrOrderFileLegacyPassPass(Registry); initializeInstrProfilingLegacyPassPass(Registry); initializeMemorySanitizerLegacyPassPass(Registry); - initializeHWAddressSanitizerPass(Registry); + initializeHWAddressSanitizerLegacyPassPass(Registry); initializeThreadSanitizerLegacyPassPass(Registry); initializeSanitizerCoverageModulePass(Registry); initializeDataFlowSanitizerPass(Registry); - initializeEfficiencySanitizerPass(Registry); } /// LLVMInitializeInstrumentation - C binding for diff --git a/lib/Transforms/Instrumentation/MaximumSpanningTree.h b/lib/Transforms/Instrumentation/MaximumSpanningTree.h index 4eb758c69c581..892a6a26da91b 100644 --- a/lib/Transforms/Instrumentation/MaximumSpanningTree.h +++ b/lib/Transforms/Instrumentation/MaximumSpanningTree.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/MaximumSpanningTree.h - Interface ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,8 +67,7 @@ namespace llvm { /// MaximumSpanningTree() - Takes a vector of weighted edges and returns a /// spanning tree. MaximumSpanningTree(EdgeWeights &EdgeVector) { - - std::stable_sort(EdgeVector.begin(), EdgeVector.end(), EdgeWeightCompare()); + llvm::stable_sort(EdgeVector, EdgeWeightCompare()); // Create spanning tree, Forest contains a special data structure // that makes checking if two nodes are already in a common (sub-)tree diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp index e6573af2077dc..b25cbed1bb021 100644 --- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -1,9 +1,8 @@ //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -144,6 +143,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -248,6 +248,13 @@ static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false)); +static cl::opt<bool> ClHandleLifetimeIntrinsics( + "msan-handle-lifetime-intrinsics", + cl::desc( + "when possible, poison scoped variables at the beginning of the scope " + "(slower, but more precise)"), + cl::Hidden, cl::init(true)); + // When compiling the Linux kernel, we sometimes see false positives related to // MSan being unable to understand that inline assembly calls may initialize // local variables. @@ -305,22 +312,23 @@ static cl::opt<bool> ClWithComdat("msan-with-comdat", // These options allow to specify custom memory map parameters // See MemoryMapParams for details. -static cl::opt<unsigned long long> ClAndMask("msan-and-mask", - cl::desc("Define custom MSan AndMask"), - cl::Hidden, cl::init(0)); +static cl::opt<uint64_t> ClAndMask("msan-and-mask", + cl::desc("Define custom MSan AndMask"), + cl::Hidden, cl::init(0)); -static cl::opt<unsigned long long> ClXorMask("msan-xor-mask", - cl::desc("Define custom MSan XorMask"), - cl::Hidden, cl::init(0)); +static cl::opt<uint64_t> ClXorMask("msan-xor-mask", + cl::desc("Define custom MSan XorMask"), + cl::Hidden, cl::init(0)); -static cl::opt<unsigned long long> ClShadowBase("msan-shadow-base", - cl::desc("Define custom MSan ShadowBase"), - cl::Hidden, cl::init(0)); +static cl::opt<uint64_t> ClShadowBase("msan-shadow-base", + cl::desc("Define custom MSan ShadowBase"), + cl::Hidden, cl::init(0)); -static cl::opt<unsigned long long> ClOriginBase("msan-origin-base", - cl::desc("Define custom MSan OriginBase"), - cl::Hidden, cl::init(0)); +static cl::opt<uint64_t> ClOriginBase("msan-origin-base", + cl::desc("Define custom MSan OriginBase"), + cl::Hidden, cl::init(0)); +static const char *const kMsanModuleCtorName = "msan.module_ctor"; static const char *const kMsanInitName = "__msan_init"; namespace { @@ -454,17 +462,16 @@ namespace { /// the module. class MemorySanitizer { public: - MemorySanitizer(Module &M, int TrackOrigins = 0, bool Recover = false, - bool EnableKmsan = false) { + MemorySanitizer(Module &M, MemorySanitizerOptions Options) { this->CompileKernel = - ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : EnableKmsan; + ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : Options.Kernel; if (ClTrackOrigins.getNumOccurrences() > 0) this->TrackOrigins = ClTrackOrigins; else - this->TrackOrigins = this->CompileKernel ? 2 : TrackOrigins; + this->TrackOrigins = this->CompileKernel ? 2 : Options.TrackOrigins; this->Recover = ClKeepGoing.getNumOccurrences() > 0 ? ClKeepGoing - : (this->CompileKernel | Recover); + : (this->CompileKernel | Options.Recover); initializeModule(M); } @@ -536,41 +543,42 @@ private: bool CallbacksInitialized = false; /// The run-time callback to print a warning. - Value *WarningFn; + FunctionCallee WarningFn; // These arrays are indexed by log2(AccessSize). - Value *MaybeWarningFn[kNumberOfAccessSizes]; - Value *MaybeStoreOriginFn[kNumberOfAccessSizes]; + FunctionCallee MaybeWarningFn[kNumberOfAccessSizes]; + FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes]; /// Run-time helper that generates a new origin value for a stack /// allocation. - Value *MsanSetAllocaOrigin4Fn; + FunctionCallee MsanSetAllocaOrigin4Fn; /// Run-time helper that poisons stack on function entry. - Value *MsanPoisonStackFn; + FunctionCallee MsanPoisonStackFn; /// Run-time helper that records a store (or any event) of an /// uninitialized value and returns an updated origin id encoding this info. - Value *MsanChainOriginFn; + FunctionCallee MsanChainOriginFn; /// MSan runtime replacements for memmove, memcpy and memset. - Value *MemmoveFn, *MemcpyFn, *MemsetFn; + FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; /// KMSAN callback for task-local function argument shadow. - Value *MsanGetContextStateFn; + StructType *MsanContextStateTy; + FunctionCallee MsanGetContextStateFn; /// Functions for poisoning/unpoisoning local variables - Value *MsanPoisonAllocaFn, *MsanUnpoisonAllocaFn; + FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn; /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin /// pointers. - Value *MsanMetadataPtrForLoadN, *MsanMetadataPtrForStoreN; - Value *MsanMetadataPtrForLoad_1_8[4]; - Value *MsanMetadataPtrForStore_1_8[4]; - Value *MsanInstrumentAsmStoreFn; + FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN; + FunctionCallee MsanMetadataPtrForLoad_1_8[4]; + FunctionCallee MsanMetadataPtrForStore_1_8[4]; + FunctionCallee MsanInstrumentAsmStoreFn; /// Helper to choose between different MsanMetadataPtrXxx(). - Value *getKmsanShadowOriginAccessFn(bool isStore, int size); + FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size); /// Memory map parameters used in application-to-shadow calculation. const MemoryMapParams *MapParams; @@ -586,6 +594,8 @@ private: /// An empty volatile inline asm that prevents callback merge. InlineAsm *EmptyAsm; + + Function *MsanCtorFunction; }; /// A legacy function pass for msan instrumentation. @@ -595,10 +605,8 @@ struct MemorySanitizerLegacyPass : public FunctionPass { // Pass identification, replacement for typeid. static char ID; - MemorySanitizerLegacyPass(int TrackOrigins = 0, bool Recover = false, - bool EnableKmsan = false) - : FunctionPass(ID), TrackOrigins(TrackOrigins), Recover(Recover), - EnableKmsan(EnableKmsan) {} + MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {}) + : FunctionPass(ID), Options(Options) {} StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -612,16 +620,14 @@ struct MemorySanitizerLegacyPass : public FunctionPass { bool doInitialization(Module &M) override; Optional<MemorySanitizer> MSan; - int TrackOrigins; - bool Recover; - bool EnableKmsan; + MemorySanitizerOptions Options; }; } // end anonymous namespace PreservedAnalyses MemorySanitizerPass::run(Function &F, FunctionAnalysisManager &FAM) { - MemorySanitizer Msan(*F.getParent(), TrackOrigins, Recover, EnableKmsan); + MemorySanitizer Msan(*F.getParent(), Options); if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) return PreservedAnalyses::none(); return PreservedAnalyses::all(); @@ -637,10 +643,9 @@ INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan", "MemorySanitizer: detects uninitialized reads.", false, false) -FunctionPass *llvm::createMemorySanitizerLegacyPassPass(int TrackOrigins, - bool Recover, - bool CompileKernel) { - return new MemorySanitizerLegacyPass(TrackOrigins, Recover, CompileKernel); +FunctionPass * +llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) { + return new MemorySanitizerLegacyPass(Options); } /// Create a non-const global initialized with the given string. @@ -675,18 +680,15 @@ void MemorySanitizer::createKernelApi(Module &M) { IRB.getInt32Ty()); // Requests the per-task context state (kmsan_context_state*) from the // runtime library. + MsanContextStateTy = StructType::get( + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), + ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */ + IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, + OriginTy); MsanGetContextStateFn = M.getOrInsertFunction( - "__msan_get_context_state", - PointerType::get( - StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), - ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), - ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), - ArrayType::get(IRB.getInt64Ty(), - kParamTLSSize / 8), /* va_arg_origin */ - IRB.getInt64Ty(), - ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, - OriginTy), - 0)); + "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0)); Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), PointerType::get(IRB.getInt32Ty(), 0)); @@ -821,8 +823,9 @@ void MemorySanitizer::initializeCallbacks(Module &M) { CallbacksInitialized = true; } -Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) { - Value **Fns = +FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, + int size) { + FunctionCallee *Fns = isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8; switch (size) { case 1: @@ -839,6 +842,8 @@ Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) { } /// Module-level initialization. +/// +/// inserts a call to __msan_init to the module's constructor list. void MemorySanitizer::initializeModule(Module &M) { auto &DL = M.getDataLayout(); @@ -913,7 +918,22 @@ void MemorySanitizer::initializeModule(Module &M) { OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000); if (!CompileKernel) { - getOrCreateInitFunction(M, kMsanInitName); + std::tie(MsanCtorFunction, std::ignore) = + getOrCreateSanitizerCtorAndInitFunctions( + M, kMsanModuleCtorName, kMsanInitName, + /*InitArgTypes=*/{}, + /*InitArgs=*/{}, + // This callback is invoked when the functions are created the first + // time. Hook them into the global ctors list in that case: + [&](Function *Ctor, FunctionCallee) { + if (!ClWithComdat) { + appendToGlobalCtors(M, Ctor, 0); + return; + } + Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); + Ctor->setComdat(MsanCtorComdat); + appendToGlobalCtors(M, Ctor, 0, Ctor); + }); if (TrackOrigins) M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] { @@ -932,7 +952,7 @@ void MemorySanitizer::initializeModule(Module &M) { } bool MemorySanitizerLegacyPass::doInitialization(Module &M) { - MSan.emplace(M, TrackOrigins, Recover, EnableKmsan); + MSan.emplace(M, Options); return true; } @@ -1011,6 +1031,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { : Shadow(S), Origin(O), OrigIns(I) {} }; SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList; + bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics; + SmallSet<AllocaInst *, 16> AllocaSet; + SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList; SmallVector<StoreInst *, 16> StoreList; MemorySanitizerVisitor(Function &F, MemorySanitizer &MS, @@ -1076,7 +1099,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) { Value *GEP = - i ? IRB.CreateConstGEP1_32(nullptr, OriginPtr, i) : OriginPtr; + i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr; IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment); CurrentAlignment = kMinOriginAlignment; } @@ -1104,7 +1127,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { - Value *Fn = MS.MaybeStoreOriginFn[SizeIndex]; + FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt( ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); IRB.CreateCall(Fn, {ConvertedShadow2, @@ -1186,7 +1209,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { - Value *Fn = MS.MaybeWarningFn[SizeIndex]; + FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex]; Value *ConvertedShadow2 = IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin @@ -1221,20 +1244,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {}); Constant *Zero = IRB.getInt32(0); - MS.ParamTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}, "param_shadow"); - MS.RetvalTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}, "retval_shadow"); - MS.VAArgTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}, "va_arg_shadow"); - MS.VAArgOriginTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}, "va_arg_origin"); - MS.VAArgOverflowSizeTLS = IRB.CreateGEP( - ContextState, {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); - MS.ParamOriginTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}, "param_origin"); + MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(0)}, "param_shadow"); + MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(1)}, "retval_shadow"); + MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(2)}, "va_arg_shadow"); + MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(3)}, "va_arg_origin"); + MS.VAArgOverflowSizeTLS = + IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); + MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(5)}, "param_origin"); MS.RetvalOriginTLS = - IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}, "retval_origin"); + IRB.CreateGEP(MS.MsanContextStateTy, ContextState, + {Zero, IRB.getInt32(6)}, "retval_origin"); return ret; } @@ -1265,6 +1290,19 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { VAHelper->finalizeInstrumentation(); + // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to + // instrumenting only allocas. + if (InstrumentLifetimeStart) { + for (auto Item : LifetimeStartList) { + instrumentAlloca(*Item.second, Item.first); + AllocaSet.erase(Item.second); + } + } + // Poison the allocas for which we didn't instrument the corresponding + // lifetime intrinsics. + for (AllocaInst *AI : AllocaSet) + instrumentAlloca(*AI); + bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 && InstrumentationList.size() + StoreList.size() > (unsigned)ClInstrumentationWithCallThreshold; @@ -1381,7 +1419,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask)); } OriginPtr = - IRB.CreateIntToPtr(OriginLong, PointerType::get(IRB.getInt32Ty(), 0)); + IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0)); } return std::make_pair(ShadowPtr, OriginPtr); } @@ -1393,7 +1431,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { const DataLayout &DL = F.getParent()->getDataLayout(); int Size = DL.getTypeStoreSize(ShadowTy); - Value *Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size); + FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size); Value *AddrCast = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); if (Getter) { @@ -1598,8 +1636,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // ParamTLS overflow. *ShadowPtr = getCleanShadow(V); } else { - *ShadowPtr = - EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment); + *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base, + kShadowTLSAlignment); } } LLVM_DEBUG(dbgs() @@ -1607,7 +1645,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (MS.TrackOrigins && !Overflow) { Value *OriginPtr = getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset); - setOrigin(A, EntryIRB.CreateLoad(OriginPtr)); + setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr)); } else { setOrigin(A, getCleanOrigin()); } @@ -1738,7 +1776,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (PropagateShadow) { std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); - setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld")); + setShadow(&I, + IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); } else { setShadow(&I, getCleanShadow(&I)); } @@ -1752,7 +1791,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (MS.TrackOrigins) { if (PropagateShadow) { unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); - setOrigin(&I, IRB.CreateAlignedLoad(OriginPtr, OriginAlignment)); + setOrigin( + &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment)); } else { setOrigin(&I, getCleanOrigin()); } @@ -1903,7 +1943,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *S1S2 = IRB.CreateAnd(S1, S2); Value *V1S2 = IRB.CreateAnd(V1, S2); Value *S1V2 = IRB.CreateAnd(S1, V2); - setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2))); + setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); setOriginForNaryOp(I); } @@ -1925,7 +1965,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *S1S2 = IRB.CreateAnd(S1, S2); Value *V1S2 = IRB.CreateAnd(V1, S2); Value *S1V2 = IRB.CreateAnd(S1, V2); - setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2))); + setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); setOriginForNaryOp(I); } @@ -2070,6 +2110,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { SC.Done(&I); } + void visitFNeg(UnaryOperator &I) { handleShadowOr(I); } + // Handle multiplication by constant. // // Handle a special case of multiplication by constant that may have one or @@ -2432,7 +2474,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { unsigned Alignment = 1; std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); - setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld")); + setShadow(&I, + IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); } else { setShadow(&I, getCleanShadow(&I)); } @@ -2442,7 +2485,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (MS.TrackOrigins) { if (PropagateShadow) - setOrigin(&I, IRB.CreateLoad(OriginPtr)); + setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr)); else setOrigin(&I, getCleanOrigin()); } @@ -2519,6 +2562,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { return false; } + void handleLifetimeStart(IntrinsicInst &I) { + if (!PoisonStack) + return; + DenseMap<Value *, AllocaInst *> AllocaForValue; + AllocaInst *AI = + llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue); + if (!AI) + InstrumentLifetimeStart = false; + LifetimeStartList.push_back(std::make_pair(&I, AI)); + } + void handleBswap(IntrinsicInst &I) { IRBuilder<> IRB(&I); Value *Op = I.getArgOperand(0); @@ -2650,7 +2704,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { : Lower64ShadowExtend(IRB, S2, getShadowTy(&I)); Value *V1 = I.getOperand(0); Value *V2 = I.getOperand(1); - Value *Shift = IRB.CreateCall(I.getCalledValue(), + Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(), {IRB.CreateBitCast(S1, V1->getType()), V2}); Shift = IRB.CreateBitCast(Shift, getShadowTy(&I)); setShadow(&I, IRB.CreateOr(Shift, S2Conv)); @@ -2660,6 +2714,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // Get an X86_MMX-sized vector type. Type *getMMXVectorTy(unsigned EltSizeInBits) { const unsigned X86_MMXSizeInBits = 64; + assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && + "Illegal MMX vector element size"); return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits), X86_MMXSizeInBits / EltSizeInBits); } @@ -2825,9 +2881,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (ClCheckAccessAddress) insertShadowCheck(Addr, &I); - Value *Shadow = IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_ldmxcsr"); - Value *Origin = - MS.TrackOrigins ? IRB.CreateLoad(OriginPtr) : getCleanOrigin(); + Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr"); + Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr) + : getCleanOrigin(); insertShadowCheck(Shadow, Origin, &I); } @@ -2901,7 +2957,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *Origin = IRB.CreateSelect( IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())), - getOrigin(PassThru), IRB.CreateLoad(OriginPtr)); + getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr)); setOrigin(&I, Origin); } else { @@ -2911,9 +2967,32 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { return true; } + // Instrument BMI / BMI2 intrinsics. + // All of these intrinsics are Z = I(X, Y) + // where the types of all operands and the result match, and are either i32 or i64. + // The following instrumentation happens to work for all of them: + // Sz = I(Sx, Y) | (sext (Sy != 0)) + void handleBmiIntrinsic(IntrinsicInst &I) { + IRBuilder<> IRB(&I); + Type *ShadowTy = getShadowTy(&I); + + // If any bit of the mask operand is poisoned, then the whole thing is. + Value *SMask = getShadow(&I, 1); + SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)), + ShadowTy); + // Apply the same intrinsic to the shadow of the first operand. + Value *S = IRB.CreateCall(I.getCalledFunction(), + {getShadow(&I, 0), I.getOperand(1)}); + S = IRB.CreateOr(SMask, S); + setShadow(&I, S); + setOriginForNaryOp(I); + } void visitIntrinsicInst(IntrinsicInst &I) { switch (I.getIntrinsicID()) { + case Intrinsic::lifetime_start: + handleLifetimeStart(I); + break; case Intrinsic::bswap: handleBswap(I); break; @@ -3127,6 +3206,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { handleVectorComparePackedIntrinsic(I); break; + case Intrinsic::x86_bmi_bextr_32: + case Intrinsic::x86_bmi_bextr_64: + case Intrinsic::x86_bmi_bzhi_32: + case Intrinsic::x86_bmi_bzhi_64: + case Intrinsic::x86_bmi_pdep_32: + case Intrinsic::x86_bmi_pdep_64: + case Intrinsic::x86_bmi_pext_32: + case Intrinsic::x86_bmi_pext_64: + handleBmiIntrinsic(I); + break; + case Intrinsic::is_constant: // The result of llvm.is.constant() is always defined. setShadow(&I, getCleanShadow(&I)); @@ -3143,21 +3233,21 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { void visitCallSite(CallSite CS) { Instruction &I = *CS.getInstruction(); assert(!I.getMetadata("nosanitize")); - assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite"); + assert((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && + "Unknown type of CallSite"); + if (CS.isCallBr() || (CS.isCall() && cast<CallInst>(&I)->isInlineAsm())) { + // For inline asm (either a call to asm function, or callbr instruction), + // do the usual thing: check argument shadow and mark all outputs as + // clean. Note that any side effects of the inline asm that are not + // immediately visible in its constraints are not handled. + if (ClHandleAsmConservative && MS.CompileKernel) + visitAsmInstruction(I); + else + visitInstruction(I); + return; + } if (CS.isCall()) { CallInst *Call = cast<CallInst>(&I); - - // For inline asm, do the usual thing: check argument shadow and mark all - // outputs as clean. Note that any side effects of the inline asm that are - // not immediately visible in its constraints are not handled. - if (Call->isInlineAsm()) { - if (ClHandleAsmConservative && MS.CompileKernel) - visitAsmInstruction(I); - else - visitInstruction(I); - return; - } - assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere"); // We are going to insert code that relies on the fact that the callee @@ -3264,12 +3354,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { "Could not find insertion point for retval shadow load"); } IRBuilder<> IRBAfter(&*NextInsn); - Value *RetvalShadow = - IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter), - kShadowTLSAlignment, "_msret"); + Value *RetvalShadow = IRBAfter.CreateAlignedLoad( + getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter), + kShadowTLSAlignment, "_msret"); setShadow(&I, RetvalShadow); if (MS.TrackOrigins) - setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter))); + setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy, + getOriginPtrForRetval(IRBAfter))); } bool isAMustTailRetVal(Value *RetVal) { @@ -3330,7 +3421,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { StackDescription.str()); } - void instrumentAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { + void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { if (PoisonStack && ClPoisonStackWithCall) { IRB.CreateCall(MS.MsanPoisonStackFn, {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); @@ -3352,7 +3443,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { } } - void instrumentAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { + void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { Value *Descr = getLocalVarDescription(I); if (PoisonStack) { IRB.CreateCall(MS.MsanPoisonAllocaFn, @@ -3364,10 +3455,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { } } - void visitAllocaInst(AllocaInst &I) { - setShadow(&I, getCleanShadow(&I)); - setOrigin(&I, getCleanOrigin()); - IRBuilder<> IRB(I.getNextNode()); + void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) { + if (!InsPoint) + InsPoint = &I; + IRBuilder<> IRB(InsPoint->getNextNode()); const DataLayout &DL = F.getParent()->getDataLayout(); uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); @@ -3375,9 +3466,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Len = IRB.CreateMul(Len, I.getArraySize()); if (MS.CompileKernel) - instrumentAllocaKmsan(I, IRB, Len); + poisonAllocaKmsan(I, IRB, Len); else - instrumentAllocaUserspace(I, IRB, Len); + poisonAllocaUserspace(I, IRB, Len); + } + + void visitAllocaInst(AllocaInst &I) { + setShadow(&I, getCleanShadow(&I)); + setOrigin(&I, getCleanOrigin()); + // We'll get to this alloca later unless it's poisoned at the corresponding + // llvm.lifetime.start. + AllocaSet.insert(&I); } void visitSelectInst(SelectInst& I) { @@ -3409,7 +3508,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { D = CreateAppToShadowCast(IRB, D); // Result shadow if condition shadow is 1. - Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd)); + Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd}); } Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select"); setShadow(&I, Sa); @@ -3525,10 +3624,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { } /// Get the number of output arguments returned by pointers. - int getNumOutputArgs(InlineAsm *IA, CallInst *CI) { + int getNumOutputArgs(InlineAsm *IA, CallBase *CB) { int NumRetOutputs = 0; int NumOutputs = 0; - Type *RetTy = dyn_cast<Value>(CI)->getType(); + Type *RetTy = dyn_cast<Value>(CB)->getType(); if (!RetTy->isVoidTy()) { // Register outputs are returned via the CallInst return value. StructType *ST = dyn_cast_or_null<StructType>(RetTy); @@ -3568,24 +3667,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // corresponding CallInst has nO+nI+1 operands (the last operand is the // function to be called). const DataLayout &DL = F.getParent()->getDataLayout(); - CallInst *CI = dyn_cast<CallInst>(&I); + CallBase *CB = dyn_cast<CallBase>(&I); IRBuilder<> IRB(&I); - InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); - int OutputArgs = getNumOutputArgs(IA, CI); + InlineAsm *IA = cast<InlineAsm>(CB->getCalledValue()); + int OutputArgs = getNumOutputArgs(IA, CB); // The last operand of a CallInst is the function itself. - int NumOperands = CI->getNumOperands() - 1; + int NumOperands = CB->getNumOperands() - 1; // Check input arguments. Doing so before unpoisoning output arguments, so // that we won't overwrite uninit values before checking them. for (int i = OutputArgs; i < NumOperands; i++) { - Value *Operand = CI->getOperand(i); + Value *Operand = CB->getOperand(i); instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false); } // Unpoison output arguments. This must happen before the actual InlineAsm // call, so that the shadow for memory published in the asm() statement // remains valid. for (int i = 0; i < OutputArgs; i++) { - Value *Operand = CI->getOperand(i); + Value *Operand = CB->getOperand(i); instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true); } @@ -3817,7 +3916,8 @@ struct VarArgAMD64Helper : public VarArgHelper { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); - VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgOverflowSize = + IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize); @@ -3836,11 +3936,13 @@ struct VarArgAMD64Helper : public VarArgHelper { IRBuilder<> IRB(OrigInst->getNextNode()); Value *VAListTag = OrigInst->getArgOperand(0); + Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, 16)), - PointerType::get(Type::getInt64PtrTy(*MS.C), 0)); - Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr); + PointerType::get(RegSaveAreaPtrTy, 0)); + Value *RegSaveAreaPtr = + IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; unsigned Alignment = 16; std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = @@ -3851,11 +3953,13 @@ struct VarArgAMD64Helper : public VarArgHelper { if (MS.TrackOrigins) IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, Alignment, AMD64FpEndOffset); + Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C); Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, 8)), - PointerType::get(Type::getInt64PtrTy(*MS.C), 0)); - Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr); + PointerType::get(OverflowArgAreaPtrTy, 0)); + Value *OverflowArgAreaPtr = + IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr); Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr; std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) = MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), @@ -3957,7 +4061,7 @@ struct VarArgMIPS64Helper : public VarArgHelper { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); - VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -3974,10 +4078,12 @@ struct VarArgMIPS64Helper : public VarArgHelper { CallInst *OrigInst = VAStartInstrumentationList[i]; IRBuilder<> IRB(OrigInst->getNextNode()); Value *VAListTag = OrigInst->getArgOperand(0); + Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), - PointerType::get(Type::getInt64PtrTy(*MS.C), 0)); - Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr); + PointerType::get(RegSaveAreaPtrTy, 0)); + Value *RegSaveAreaPtr = + IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; unsigned Alignment = 8; std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = @@ -4127,7 +4233,7 @@ struct VarArgAArch64Helper : public VarArgHelper { IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, offset)), Type::getInt64PtrTy(*MS.C)); - return IRB.CreateLoad(SaveAreaPtrPtr); + return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr); } // Retrieve a va_list field of 'int' size. @@ -4137,7 +4243,7 @@ struct VarArgAArch64Helper : public VarArgHelper { IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), ConstantInt::get(MS.IntptrTy, offset)), Type::getInt32PtrTy(*MS.C)); - Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr); + Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr); return IRB.CreateSExt(SaveArea32, MS.IntptrTy); } @@ -4148,7 +4254,8 @@ struct VarArgAArch64Helper : public VarArgHelper { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); - VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgOverflowSize = + IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize); @@ -4391,7 +4498,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper { assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); - VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -4408,10 +4515,12 @@ struct VarArgPowerPC64Helper : public VarArgHelper { CallInst *OrigInst = VAStartInstrumentationList[i]; IRBuilder<> IRB(OrigInst->getNextNode()); Value *VAListTag = OrigInst->getArgOperand(0); + Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), - PointerType::get(Type::getInt64PtrTy(*MS.C), 0)); - Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr); + PointerType::get(RegSaveAreaPtrTy, 0)); + Value *RegSaveAreaPtr = + IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; unsigned Alignment = 8; std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = @@ -4458,6 +4567,8 @@ static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, } bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) { + if (!CompileKernel && (&F == MsanCtorFunction)) + return false; MemorySanitizerVisitor Visitor(F, *this, TLI); // Clear out readonly/readnone attributes. diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index f043325f5bba1..6fec3c9c79ee3 100644 --- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1,9 +1,8 @@ //===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -48,7 +47,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "CFGMST.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" @@ -66,6 +64,7 @@ #include "llvm/Analysis/IndirectCallVisitor.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -107,6 +106,7 @@ #include "llvm/Support/JamCRC.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include <algorithm> #include <cassert> @@ -133,6 +133,19 @@ STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts."); STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); +STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO."); +STATISTIC(NumOfCSPGOSelectInsts, + "Number of select instruction instrumented in CSPGO."); +STATISTIC(NumOfCSPGOMemIntrinsics, + "Number of mem intrinsics instrumented in CSPGO."); +STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO."); +STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO."); +STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO."); +STATISTIC(NumOfCSPGOFunc, + "Number of functions having valid profile counts in CSPGO."); +STATISTIC(NumOfCSPGOMismatch, + "Number of functions having mismatch profile in CSPGO."); +STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO."); // Command line option to specify the file to read profile from. This is // mainly used for testing. @@ -384,7 +397,8 @@ class PGOInstrumentationGenLegacyPass : public ModulePass { public: static char ID; - PGOInstrumentationGenLegacyPass() : ModulePass(ID) { + PGOInstrumentationGenLegacyPass(bool IsCS = false) + : ModulePass(ID), IsCS(IsCS) { initializePGOInstrumentationGenLegacyPassPass( *PassRegistry::getPassRegistry()); } @@ -392,6 +406,8 @@ public: StringRef getPassName() const override { return "PGOInstrumentationGenPass"; } private: + // Is this is context-sensitive instrumentation. + bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -404,8 +420,8 @@ public: static char ID; // Provide the profile filename as the parameter. - PGOInstrumentationUseLegacyPass(std::string Filename = "") - : ModulePass(ID), ProfileFileName(std::move(Filename)) { + PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false) + : ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; initializePGOInstrumentationUseLegacyPassPass( @@ -416,14 +432,38 @@ public: private: std::string ProfileFileName; + // Is this is context-sensitive instrumentation use. + bool IsCS; bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<ProfileSummaryInfoWrapperPass>(); AU.addRequired<BlockFrequencyInfoWrapperPass>(); } }; +class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass { +public: + static char ID; + StringRef getPassName() const override { + return "PGOInstrumentationGenCreateVarPass"; + } + PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "") + : ModulePass(ID), InstrProfileOutput(CSInstrName) { + initializePGOInstrumentationGenCreateVarLegacyPassPass( + *PassRegistry::getPassRegistry()); + } + +private: + bool runOnModule(Module &M) override { + createProfileFileNameVar(M, InstrProfileOutput); + createIRLevelProfileFlagVar(M, true); + return false; + } + std::string InstrProfileOutput; +}; + } // end anonymous namespace char PGOInstrumentationGenLegacyPass::ID = 0; @@ -435,8 +475,8 @@ INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) -ModulePass *llvm::createPGOInstrumentationGenLegacyPass() { - return new PGOInstrumentationGenLegacyPass(); +ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) { + return new PGOInstrumentationGenLegacyPass(IsCS); } char PGOInstrumentationUseLegacyPass::ID = 0; @@ -445,11 +485,25 @@ INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) -ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) { - return new PGOInstrumentationUseLegacyPass(Filename.str()); +ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename, + bool IsCS) { + return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS); +} + +char PGOInstrumentationGenCreateVarLegacyPass::ID = 0; + +INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass, + "pgo-instr-gen-create-var", + "Create PGO instrumentation version variable for CSPGO.", false, + false) + +ModulePass * +llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) { + return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName); } namespace { @@ -490,6 +544,12 @@ struct BBInfo { const std::string infoString() const { return (Twine("Index=") + Twine(Index)).str(); } + + // Empty function -- only applicable to UseBBInfo. + void addOutEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {} + + // Empty function -- only applicable to UseBBInfo. + void addInEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {} }; // This class implements the CFG edges. Note the CFG can be a multi-graph. @@ -497,6 +557,9 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation { private: Function &F; + // Is this is context-sensitive instrumentation. + bool IsCS; + // A map that stores the Comdat group in function F. std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers; @@ -516,6 +579,10 @@ public: // The Minimum Spanning Tree of function CFG. CFGMST<Edge, BBInfo> MST; + // Collect all the BBs that will be instrumented, and store them in + // InstrumentBBs. + void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs); + // Give an edge, find the BB that will be instrumented. // Return nullptr if there is no BB to be instrumented. BasicBlock *getInstrBB(Edge *E); @@ -536,15 +603,23 @@ public: Function &Func, std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFI = nullptr) - : F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1), - SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) { + BlockFrequencyInfo *BFI = nullptr, bool IsCS = false) + : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), + ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func), + MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); MIVisitor.countMemIntrinsics(Func); - NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); - NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); - ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func); + if (!IsCS) { + NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); + NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); + NumOfPGOBB += MST.BBInfos.size(); + ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func); + } else { + NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts(); + NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics(); + NumOfCSPGOBB += MST.BBInfos.size(); + } ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func); FuncName = getPGOFuncName(F); @@ -553,28 +628,17 @@ public: renameComdatFunction(); LLVM_DEBUG(dumpInfo("after CFGMST")); - NumOfPGOBB += MST.BBInfos.size(); for (auto &E : MST.AllEdges) { if (E->Removed) continue; - NumOfPGOEdge++; + IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++; if (!E->InMST) - NumOfPGOInstrument++; + IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++; } if (CreateGlobalVar) FuncNameVar = createPGOFuncNameVar(F, FuncName); } - - // Return the number of profile counters needed for the function. - unsigned getNumCounters() { - unsigned NumCounters = 0; - for (auto &E : this->MST.AllEdges) { - if (!E->InMST && !E->Removed) - NumCounters++; - } - return NumCounters + SIVisitor.getNumOfSelectInsts(); - } }; } // end anonymous namespace @@ -598,9 +662,17 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() { } } JC.update(Indexes); + + // Hash format for context sensitive profile. Reserve 4 bits for other + // information. FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 | (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 | + //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 | (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC(); + // Reserve bit 60-63 for other information purpose. + FunctionHash &= 0x0FFFFFFFFFFFFFFF; + if (IsCS) + NamedInstrProfRecord::setCSFlagInHash(FunctionHash); LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n" << " CRC = " << JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts() @@ -681,6 +753,36 @@ void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() { } } +// Collect all the BBs that will be instruments and return them in +// InstrumentBBs and setup InEdges/OutEdge for UseBBInfo. +template <class Edge, class BBInfo> +void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs( + std::vector<BasicBlock *> &InstrumentBBs) { + // Use a worklist as we will update the vector during the iteration. + std::vector<Edge *> EdgeList; + EdgeList.reserve(MST.AllEdges.size()); + for (auto &E : MST.AllEdges) + EdgeList.push_back(E.get()); + + for (auto &E : EdgeList) { + BasicBlock *InstrBB = getInstrBB(E); + if (InstrBB) + InstrumentBBs.push_back(InstrBB); + } + + // Set up InEdges/OutEdges for all BBs. + for (auto &E : MST.AllEdges) { + if (E->Removed) + continue; + const BasicBlock *SrcBB = E->SrcBB; + const BasicBlock *DestBB = E->DestBB; + BBInfo &SrcInfo = getBBInfo(SrcBB); + BBInfo &DestInfo = getBBInfo(DestBB); + SrcInfo.addOutEdge(E.get()); + DestInfo.addInEdge(E.get()); + } +} + // Given a CFG E to be instrumented, find which BB to place the instrumented // code. The function will split the critical edge if necessary. template <class Edge, class BBInfo> @@ -696,46 +798,64 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) { if (DestBB == nullptr) return SrcBB; + auto canInstrument = [](BasicBlock *BB) -> BasicBlock * { + // There are basic blocks (such as catchswitch) cannot be instrumented. + // If the returned first insertion point is the end of BB, skip this BB. + if (BB->getFirstInsertionPt() == BB->end()) + return nullptr; + return BB; + }; + // Instrument the SrcBB if it has a single successor, // otherwise, the DestBB if this is not a critical edge. Instruction *TI = SrcBB->getTerminator(); if (TI->getNumSuccessors() <= 1) - return SrcBB; + return canInstrument(SrcBB); if (!E->IsCritical) - return DestBB; + return canInstrument(DestBB); + unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); + BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum); + if (!InstrBB) { + LLVM_DEBUG( + dbgs() << "Fail to split critical edge: not instrument this edge.\n"); + return nullptr; + } // For a critical edge, we have to split. Instrument the newly // created BB. - NumOfPGOSplit++; + IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++; LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> " << getBBInfo(DestBB).Index << "\n"); - unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); - BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum); - assert(InstrBB && "Critical edge is not split"); - + // Need to add two new edges. First one: Add new edge of SrcBB->InstrBB. + MST.addEdge(SrcBB, InstrBB, 0); + // Second one: Add new edge of InstrBB->DestBB. + Edge &NewEdge1 = MST.addEdge(InstrBB, DestBB, 0); + NewEdge1.InMST = true; E->Removed = true; - return InstrBB; + + return canInstrument(InstrBB); } // Visit all edge and instrument the edges not in MST, and do value profiling. // Critical edges will be split. static void instrumentOneFunc( Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, - std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) { + std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, + bool IsCS) { // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); + FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI, - BFI); - unsigned NumCounters = FuncInfo.getNumCounters(); + BFI, IsCS); + std::vector<BasicBlock *> InstrumentBBs; + FuncInfo.getInstrumentBBs(InstrumentBBs); + unsigned NumCounters = + InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts(); uint32_t I = 0; Type *I8PtrTy = Type::getInt8PtrTy(M->getContext()); - for (auto &E : FuncInfo.MST.AllEdges) { - BasicBlock *InstrBB = FuncInfo.getInstrBB(E.get()); - if (!InstrBB) - continue; - + for (auto *InstrBB : InstrumentBBs) { IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt()); assert(Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"); @@ -831,6 +951,18 @@ struct UseBBInfo : public BBInfo { return BBInfo::infoString(); return (Twine(BBInfo::infoString()) + " Count=" + Twine(CountValue)).str(); } + + // Add an OutEdge and update the edge count. + void addOutEdge(PGOUseEdge *E) { + OutEdges.push_back(E); + UnknownCountOutEdge++; + } + + // Add an InEdge and update the edge count. + void addInEdge(PGOUseEdge *E) { + InEdges.push_back(E); + UnknownCountInEdge++; + } }; } // end anonymous namespace @@ -853,10 +985,10 @@ public: PGOUseFunc(Function &Func, Module *Modu, std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFIin = nullptr) + BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false) : F(Func), M(Modu), BFI(BFIin), - FuncInfo(Func, ComdatMembers, false, BPI, BFIin), - FreqAttr(FFA_Normal) {} + FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS), + FreqAttr(FFA_Normal), IsCS(IsCS) {} // Read counts for the instrumented BB from profile. bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros); @@ -929,8 +1061,11 @@ private: // Function hotness info derived from profile. FuncFreqAttr FreqAttr; - // Find the Instrumented BB and set the value. - void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile); + // Is to use the context sensitive profile. + bool IsCS; + + // Find the Instrumented BB and set the value. Return false on error. + bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile); // Set the edge counter value for the unknown edge -- there should be only // one unknown edge. @@ -959,41 +1094,64 @@ private: } // end anonymous namespace // Visit all the edges and assign the count value for the instrumented -// edges and the BB. -void PGOUseFunc::setInstrumentedCounts( +// edges and the BB. Return false on error. +bool PGOUseFunc::setInstrumentedCounts( const std::vector<uint64_t> &CountFromProfile) { - assert(FuncInfo.getNumCounters() == CountFromProfile.size()); - // Use a worklist as we will update the vector during the iteration. - std::vector<PGOUseEdge *> WorkList; - for (auto &E : FuncInfo.MST.AllEdges) - WorkList.push_back(E.get()); + std::vector<BasicBlock *> InstrumentBBs; + FuncInfo.getInstrumentBBs(InstrumentBBs); + unsigned NumCounters = + InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts(); + // The number of counters here should match the number of counters + // in profile. Return if they mismatch. + if (NumCounters != CountFromProfile.size()) { + return false; + } + // Set the profile count to the Instrumented BBs. uint32_t I = 0; - for (auto &E : WorkList) { - BasicBlock *InstrBB = FuncInfo.getInstrBB(E); - if (!InstrBB) - continue; + for (BasicBlock *InstrBB : InstrumentBBs) { uint64_t CountValue = CountFromProfile[I++]; - if (!E->Removed) { - getBBInfo(InstrBB).setBBInfoCount(CountValue); - E->setEdgeCount(CountValue); - continue; - } - - // Need to add two new edges. - BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB); - BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB); - // Add new edge of SrcBB->InstrBB. - PGOUseEdge &NewEdge = FuncInfo.MST.addEdge(SrcBB, InstrBB, 0); - NewEdge.setEdgeCount(CountValue); - // Add new edge of InstrBB->DestBB. - PGOUseEdge &NewEdge1 = FuncInfo.MST.addEdge(InstrBB, DestBB, 0); - NewEdge1.setEdgeCount(CountValue); - NewEdge1.InMST = true; - getBBInfo(InstrBB).setBBInfoCount(CountValue); + UseBBInfo &Info = getBBInfo(InstrBB); + Info.setBBInfoCount(CountValue); } ProfileCountSize = CountFromProfile.size(); CountPosition = I; + + // Set the edge count and update the count of unknown edges for BBs. + auto setEdgeCount = [this](PGOUseEdge *E, uint64_t Value) -> void { + E->setEdgeCount(Value); + this->getBBInfo(E->SrcBB).UnknownCountOutEdge--; + this->getBBInfo(E->DestBB).UnknownCountInEdge--; + }; + + // Set the profile count the Instrumented edges. There are BBs that not in + // MST but not instrumented. Need to set the edge count value so that we can + // populate the profile counts later. + for (auto &E : FuncInfo.MST.AllEdges) { + if (E->Removed || E->InMST) + continue; + const BasicBlock *SrcBB = E->SrcBB; + UseBBInfo &SrcInfo = getBBInfo(SrcBB); + + // If only one out-edge, the edge profile count should be the same as BB + // profile count. + if (SrcInfo.CountValid && SrcInfo.OutEdges.size() == 1) + setEdgeCount(E.get(), SrcInfo.CountValue); + else { + const BasicBlock *DestBB = E->DestBB; + UseBBInfo &DestInfo = getBBInfo(DestBB); + // If only one in-edge, the edge profile count should be the same as BB + // profile count. + if (DestInfo.CountValid && DestInfo.InEdges.size() == 1) + setEdgeCount(E.get(), DestInfo.CountValue); + } + if (E->CountValid) + continue; + // E's count should have been set from profile. If not, this meenas E skips + // the instrumentation. We set the count to 0. + setEdgeCount(E.get(), 0); + } + return true; } // Set the count value for the unknown edge. There should be one and only one @@ -1022,23 +1180,31 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { auto Err = IPE.get(); bool SkipWarning = false; + LLVM_DEBUG(dbgs() << "Error in reading profile for Func " + << FuncInfo.FuncName << ": "); if (Err == instrprof_error::unknown_function) { - NumOfPGOMissing++; + IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++; SkipWarning = !PGOWarnMissing; + LLVM_DEBUG(dbgs() << "unknown function"); } else if (Err == instrprof_error::hash_mismatch || Err == instrprof_error::malformed) { - NumOfPGOMismatch++; + IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++; SkipWarning = NoPGOWarnMismatch || (NoPGOWarnMismatchComdat && (F.hasComdat() || F.getLinkage() == GlobalValue::AvailableExternallyLinkage)); + LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")"); } + LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n"); if (SkipWarning) return; - std::string Msg = IPE.message() + std::string(" ") + F.getName().str(); + std::string Msg = IPE.message() + std::string(" ") + F.getName().str() + + std::string(" Hash = ") + + std::to_string(FuncInfo.FunctionHash); + Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); }); @@ -1047,7 +1213,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) ProfileRecord = std::move(Result.get()); std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts; - NumOfPGOFunc++; + IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++; LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n"); uint64_t ValueSum = 0; for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) { @@ -1061,34 +1227,23 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) getBBInfo(nullptr).UnknownCountOutEdge = 2; getBBInfo(nullptr).UnknownCountInEdge = 2; - setInstrumentedCounts(CountFromProfile); - ProgramMaxCount = PGOReader->getMaximumFunctionCount(); + if (!setInstrumentedCounts(CountFromProfile)) { + LLVM_DEBUG( + dbgs() << "Inconsistent number of counts, skipping this function"); + Ctx.diagnose(DiagnosticInfoPGOProfile( + M->getName().data(), + Twine("Inconsistent number of counts in ") + F.getName().str() + + Twine(": the profile may be stale or there is a function name collision."), + DS_Warning)); + return false; + } + ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS); return true; } // Populate the counters from instrumented BBs to all BBs. // In the end of this operation, all BBs should have a valid count value. void PGOUseFunc::populateCounters() { - // First set up Count variable for all BBs. - for (auto &E : FuncInfo.MST.AllEdges) { - if (E->Removed) - continue; - - const BasicBlock *SrcBB = E->SrcBB; - const BasicBlock *DestBB = E->DestBB; - UseBBInfo &SrcInfo = getBBInfo(SrcBB); - UseBBInfo &DestInfo = getBBInfo(DestBB); - SrcInfo.OutEdges.push_back(E.get()); - DestInfo.InEdges.push_back(E.get()); - SrcInfo.UnknownCountOutEdge++; - DestInfo.UnknownCountInEdge++; - - if (!E->CountValid) - continue; - DestInfo.UnknownCountInEdge--; - SrcInfo.UnknownCountOutEdge--; - } - bool Changes = true; unsigned NumPasses = 0; while (Changes) { @@ -1167,7 +1322,8 @@ void PGOUseFunc::populateCounters() { // Assign the scaled count values to the BB with multiple out edges. void PGOUseFunc::setBranchWeights() { // Generate MD_prof metadata for every branch instruction. - LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n"); + LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName() + << " IsCS=" << IsCS << "\n"); for (auto &BB : F) { Instruction *TI = BB.getTerminator(); if (TI->getNumSuccessors() < 2) @@ -1175,6 +1331,7 @@ void PGOUseFunc::setBranchWeights() { if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) || isa<IndirectBrInst>(TI))) continue; + if (getBBInfo(&BB).CountValue == 0) continue; @@ -1282,7 +1439,7 @@ void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) { Type *Int64Ty = Builder.getInt64Ty(); Type *I8PtrTy = Builder.getInt8PtrTy(); Value *Length = MI.getLength(); - assert(!dyn_cast<ConstantInt>(Length)); + assert(!isa<ConstantInt>(Length)); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), @@ -1325,8 +1482,14 @@ void PGOUseFunc::annotateValueSites() { annotateValueSites(Kind); } +static const char *ValueProfKindDescr[] = { +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr, +#include "llvm/ProfileData/InstrProfData.inc" +}; + // Annotate the instructions for a specific value kind. void PGOUseFunc::annotateValueSites(uint32_t Kind) { + assert(Kind <= IPVK_Last); unsigned ValueSiteIndex = 0; auto &ValueSites = FuncInfo.ValueSites[Kind]; unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind); @@ -1334,8 +1497,10 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) { auto &Ctx = M->getContext(); Ctx.diagnose(DiagnosticInfoPGOProfile( M->getName().data(), - Twine("Inconsistent number of value sites for kind = ") + Twine(Kind) + - " in " + F.getName().str(), + Twine("Inconsistent number of value sites for ") + + Twine(ValueProfKindDescr[Kind]) + + Twine(" profiling in \"") + F.getName().str() + + Twine("\", possibly due to the use of a stale profile."), DS_Warning)); return; } @@ -1352,24 +1517,6 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) { } } -// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime -// aware this is an ir_level profile so it can set the version flag. -static void createIRLevelProfileFlagVariable(Module &M) { - Type *IntTy64 = Type::getInt64Ty(M.getContext()); - uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); - auto IRLevelVersionVariable = new GlobalVariable( - M, IntTy64, true, GlobalVariable::ExternalLinkage, - Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), - INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); - IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); - Triple TT(M.getTargetTriple()); - if (!TT.supportsCOMDAT()) - IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage); - else - IRLevelVersionVariable->setComdat(M.getOrInsertComdat( - StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)))); -} - // Collect the set of members for each Comdat in module M and store // in ComdatMembers. static void collectComdatMembers( @@ -1390,8 +1537,11 @@ static void collectComdatMembers( static bool InstrumentAllFunctions( Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, - function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { - createIRLevelProfileFlagVariable(M); + function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) { + // For the context-sensitve instrumentation, we should have a separated pass + // (before LTO/ThinLTO linking) to create these variables. + if (!IsCS) + createIRLevelProfileFlagVar(M, /* IsCS */ false); std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers; collectComdatMembers(M, ComdatMembers); @@ -1400,11 +1550,18 @@ static bool InstrumentAllFunctions( continue; auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); - instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers); + instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS); } return true; } +PreservedAnalyses +PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) { + createProfileFileNameVar(M, CSInstrName); + createIRLevelProfileFlagVar(M, /* IsCS */ true); + return PreservedAnalyses::all(); +} + bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { if (skipModule(M)) return false; @@ -1415,7 +1572,7 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { auto LookupBFI = [this](Function &F) { return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); }; - return InstrumentAllFunctions(M, LookupBPI, LookupBFI); + return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS); } PreservedAnalyses PGOInstrumentationGen::run(Module &M, @@ -1429,7 +1586,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M, return &FAM.getResult<BlockFrequencyAnalysis>(F); }; - if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI)) + if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1438,7 +1595,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M, static bool annotateAllFunctions( Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, - function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { + function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. @@ -1459,6 +1616,9 @@ static bool annotateAllFunctions( StringRef("Cannot get PGOReader"))); return false; } + if (!PGOReader->hasCSIRLevelProfile() && IsCS) + return false; + // TODO: might need to change the warning once the clang option is finalized. if (!PGOReader->isIRLevelProfile()) { Ctx.diagnose(DiagnosticInfoPGOProfile( @@ -1478,7 +1638,7 @@ static bool annotateAllFunctions( // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI); + PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS); bool AllZeros = false; if (!Func.readCounters(PGOReader.get(), AllZeros)) continue; @@ -1526,7 +1686,10 @@ static bool annotateAllFunctions( } } } - M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext())); + M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), + IsCS ? ProfileSummary::PSK_CSInstr + : ProfileSummary::PSK_Instr); + // Set function hotness attribute from the profile. // We have to apply these attributes at the end because their presence // can affect the BranchProbabilityInfo of any callers, resulting in an @@ -1545,9 +1708,10 @@ static bool annotateAllFunctions( } PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename, - std::string RemappingFilename) + std::string RemappingFilename, + bool IsCS) : ProfileFileName(std::move(Filename)), - ProfileRemappingFileName(std::move(RemappingFilename)) { + ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; if (!PGOTestProfileRemappingFile.empty()) @@ -1567,7 +1731,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, }; if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, - LookupBPI, LookupBFI)) + LookupBPI, LookupBFI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1584,7 +1748,8 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); }; - return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI); + return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, + IsCS); } static std::string getSimpleNodeName(const BasicBlock *Node) { diff --git a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp index 2c71e75dadcc4..188f95b4676bb 100644 --- a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp +++ b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp @@ -1,9 +1,8 @@ //===-- PGOMemOPSizeOpt.cpp - Optimizations based on value profiling ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,12 +19,12 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/DomTreeUpdater.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" diff --git a/lib/Transforms/Instrumentation/PoisonChecking.cpp b/lib/Transforms/Instrumentation/PoisonChecking.cpp new file mode 100644 index 0000000000000..81d92e724c7d2 --- /dev/null +++ b/lib/Transforms/Instrumentation/PoisonChecking.cpp @@ -0,0 +1,357 @@ +//===- PoisonChecking.cpp - -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements a transform pass which instruments IR such that poison semantics +// are made explicit. That is, it provides a (possibly partial) executable +// semantics for every instruction w.r.t. poison as specified in the LLVM +// LangRef. There are obvious parallels to the sanitizer tools, but this pass +// is focused purely on the semantics of LLVM IR, not any particular source +// language. If you're looking for something to see if your C/C++ contains +// UB, this is not it. +// +// The rewritten semantics of each instruction will include the following +// components: +// +// 1) The original instruction, unmodified. +// 2) A propagation rule which translates dynamic information about the poison +// state of each input to whether the dynamic output of the instruction +// produces poison. +// 3) A flag validation rule which validates any poison producing flags on the +// instruction itself (e.g. checks for overflow on nsw). +// 4) A check rule which traps (to a handler function) if this instruction must +// execute undefined behavior given the poison state of it's inputs. +// +// At the moment, the UB detection is done in a best effort manner; that is, +// the resulting code may produce a false negative result (not report UB when +// it actually exists according to the LangRef spec), but should never produce +// a false positive (report UB where it doesn't exist). The intention is to +// eventually support a "strict" mode which never dynamically reports a false +// negative at the cost of rejecting some valid inputs to translation. +// +// Use cases for this pass include: +// - Understanding (and testing!) the implications of the definition of poison +// from the LangRef. +// - Validating the output of a IR fuzzer to ensure that all programs produced +// are well defined on the specific input used. +// - Finding/confirming poison specific miscompiles by checking the poison +// status of an input/IR pair is the same before and after an optimization +// transform. +// - Checking that a bugpoint reduction does not introduce UB which didn't +// exist in the original program being reduced. +// +// The major sources of inaccuracy are currently: +// - Most validation rules not yet implemented for instructions with poison +// relavant flags. At the moment, only nsw/nuw on add/sub are supported. +// - UB which is control dependent on a branch on poison is not yet +// reported. Currently, only data flow dependence is modeled. +// - Poison which is propagated through memory is not modeled. As such, +// storing poison to memory and then reloading it will cause a false negative +// as we consider the reloaded value to not be poisoned. +// - Poison propagation across function boundaries is not modeled. At the +// moment, all arguments and return values are assumed not to be poison. +// - Undef is not modeled. In particular, the optimizer's freedom to pick +// concrete values for undef bits so as to maximize potential for producing +// poison is not modeled. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/PoisonChecking.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/MemoryBuiltins.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/PatternMatch.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "poison-checking" + +static cl::opt<bool> +LocalCheck("poison-checking-function-local", + cl::init(false), + cl::desc("Check that returns are non-poison (for testing)")); + + +static bool isConstantFalse(Value* V) { + assert(V->getType()->isIntegerTy(1)); + if (auto *CI = dyn_cast<ConstantInt>(V)) + return CI->isZero(); + return false; +} + +static Value *buildOrChain(IRBuilder<> &B, ArrayRef<Value*> Ops) { + if (Ops.size() == 0) + return B.getFalse(); + unsigned i = 0; + for (; i < Ops.size() && isConstantFalse(Ops[i]); i++) {} + if (i == Ops.size()) + return B.getFalse(); + Value *Accum = Ops[i++]; + for (; i < Ops.size(); i++) + if (!isConstantFalse(Ops[i])) + Accum = B.CreateOr(Accum, Ops[i]); + return Accum; +} + +static void generatePoisonChecksForBinOp(Instruction &I, + SmallVector<Value*, 2> &Checks) { + assert(isa<BinaryOperator>(I)); + + IRBuilder<> B(&I); + Value *LHS = I.getOperand(0); + Value *RHS = I.getOperand(1); + switch (I.getOpcode()) { + default: + return; + case Instruction::Add: { + if (I.hasNoSignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + if (I.hasNoUnsignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + break; + } + case Instruction::Sub: { + if (I.hasNoSignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::ssub_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + if (I.hasNoUnsignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::usub_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + break; + } + case Instruction::Mul: { + if (I.hasNoSignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::smul_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + if (I.hasNoUnsignedWrap()) { + auto *OverflowOp = + B.CreateBinaryIntrinsic(Intrinsic::umul_with_overflow, LHS, RHS); + Checks.push_back(B.CreateExtractValue(OverflowOp, 1)); + } + break; + } + case Instruction::UDiv: { + if (I.isExact()) { + auto *Check = + B.CreateICmp(ICmpInst::ICMP_NE, B.CreateURem(LHS, RHS), + ConstantInt::get(LHS->getType(), 0)); + Checks.push_back(Check); + } + break; + } + case Instruction::SDiv: { + if (I.isExact()) { + auto *Check = + B.CreateICmp(ICmpInst::ICMP_NE, B.CreateSRem(LHS, RHS), + ConstantInt::get(LHS->getType(), 0)); + Checks.push_back(Check); + } + break; + } + case Instruction::AShr: + case Instruction::LShr: + case Instruction::Shl: { + Value *ShiftCheck = + B.CreateICmp(ICmpInst::ICMP_UGE, RHS, + ConstantInt::get(RHS->getType(), + LHS->getType()->getScalarSizeInBits())); + Checks.push_back(ShiftCheck); + break; + } + }; +} + +static Value* generatePoisonChecks(Instruction &I) { + IRBuilder<> B(&I); + SmallVector<Value*, 2> Checks; + if (isa<BinaryOperator>(I) && !I.getType()->isVectorTy()) + generatePoisonChecksForBinOp(I, Checks); + + // Handle non-binops seperately + switch (I.getOpcode()) { + default: + break; + case Instruction::ExtractElement: { + Value *Vec = I.getOperand(0); + if (Vec->getType()->getVectorIsScalable()) + break; + Value *Idx = I.getOperand(1); + unsigned NumElts = Vec->getType()->getVectorNumElements(); + Value *Check = + B.CreateICmp(ICmpInst::ICMP_UGE, Idx, + ConstantInt::get(Idx->getType(), NumElts)); + Checks.push_back(Check); + break; + } + case Instruction::InsertElement: { + Value *Vec = I.getOperand(0); + if (Vec->getType()->getVectorIsScalable()) + break; + Value *Idx = I.getOperand(2); + unsigned NumElts = Vec->getType()->getVectorNumElements(); + Value *Check = + B.CreateICmp(ICmpInst::ICMP_UGE, Idx, + ConstantInt::get(Idx->getType(), NumElts)); + Checks.push_back(Check); + break; + } + }; + return buildOrChain(B, Checks); +} + +static Value *getPoisonFor(DenseMap<Value *, Value *> &ValToPoison, Value *V) { + auto Itr = ValToPoison.find(V); + if (Itr != ValToPoison.end()) + return Itr->second; + if (isa<Constant>(V)) { + return ConstantInt::getFalse(V->getContext()); + } + // Return false for unknwon values - this implements a non-strict mode where + // unhandled IR constructs are simply considered to never produce poison. At + // some point in the future, we probably want a "strict mode" for testing if + // nothing else. + return ConstantInt::getFalse(V->getContext()); +} + +static void CreateAssert(IRBuilder<> &B, Value *Cond) { + assert(Cond->getType()->isIntegerTy(1)); + if (auto *CI = dyn_cast<ConstantInt>(Cond)) + if (CI->isAllOnesValue()) + return; + + Module *M = B.GetInsertBlock()->getModule(); + M->getOrInsertFunction("__poison_checker_assert", + Type::getVoidTy(M->getContext()), + Type::getInt1Ty(M->getContext())); + Function *TrapFunc = M->getFunction("__poison_checker_assert"); + B.CreateCall(TrapFunc, Cond); +} + +static void CreateAssertNot(IRBuilder<> &B, Value *Cond) { + assert(Cond->getType()->isIntegerTy(1)); + CreateAssert(B, B.CreateNot(Cond)); +} + +static bool rewrite(Function &F) { + auto * const Int1Ty = Type::getInt1Ty(F.getContext()); + + DenseMap<Value *, Value *> ValToPoison; + + for (BasicBlock &BB : F) + for (auto I = BB.begin(); isa<PHINode>(&*I); I++) { + auto *OldPHI = cast<PHINode>(&*I); + auto *NewPHI = PHINode::Create(Int1Ty, + OldPHI->getNumIncomingValues()); + for (unsigned i = 0; i < OldPHI->getNumIncomingValues(); i++) + NewPHI->addIncoming(UndefValue::get(Int1Ty), + OldPHI->getIncomingBlock(i)); + NewPHI->insertBefore(OldPHI); + ValToPoison[OldPHI] = NewPHI; + } + + for (BasicBlock &BB : F) + for (Instruction &I : BB) { + if (isa<PHINode>(I)) continue; + + IRBuilder<> B(cast<Instruction>(&I)); + + // Note: There are many more sources of documented UB, but this pass only + // attempts to find UB triggered by propagation of poison. + if (Value *Op = const_cast<Value*>(getGuaranteedNonFullPoisonOp(&I))) + CreateAssertNot(B, getPoisonFor(ValToPoison, Op)); + + if (LocalCheck) + if (auto *RI = dyn_cast<ReturnInst>(&I)) + if (RI->getNumOperands() != 0) { + Value *Op = RI->getOperand(0); + CreateAssertNot(B, getPoisonFor(ValToPoison, Op)); + } + + SmallVector<Value*, 4> Checks; + if (propagatesFullPoison(&I)) + for (Value *V : I.operands()) + Checks.push_back(getPoisonFor(ValToPoison, V)); + + if (auto *Check = generatePoisonChecks(I)) + Checks.push_back(Check); + ValToPoison[&I] = buildOrChain(B, Checks); + } + + for (BasicBlock &BB : F) + for (auto I = BB.begin(); isa<PHINode>(&*I); I++) { + auto *OldPHI = cast<PHINode>(&*I); + if (!ValToPoison.count(OldPHI)) + continue; // skip the newly inserted phis + auto *NewPHI = cast<PHINode>(ValToPoison[OldPHI]); + for (unsigned i = 0; i < OldPHI->getNumIncomingValues(); i++) { + auto *OldVal = OldPHI->getIncomingValue(i); + NewPHI->setIncomingValue(i, getPoisonFor(ValToPoison, OldVal)); + } + } + return true; +} + + +PreservedAnalyses PoisonCheckingPass::run(Module &M, + ModuleAnalysisManager &AM) { + bool Changed = false; + for (auto &F : M) + Changed |= rewrite(F); + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + +PreservedAnalyses PoisonCheckingPass::run(Function &F, + FunctionAnalysisManager &AM) { + return rewrite(F) ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + + +/* Major TODO Items: + - Control dependent poison UB + - Strict mode - (i.e. must analyze every operand) + - Poison through memory + - Function ABIs + - Full coverage of intrinsics, etc.. (ouch) + + Instructions w/Unclear Semantics: + - shufflevector - It would seem reasonable for an out of bounds mask element + to produce poison, but the LangRef does not state. + - and/or - It would seem reasonable for poison to propagate from both + arguments, but LangRef doesn't state and propagatesFullPoison doesn't + include these two. + - all binary ops w/vector operands - The likely interpretation would be that + any element overflowing should produce poison for the entire result, but + the LangRef does not state. + - Floating point binary ops w/fmf flags other than (nnan, noinfs). It seems + strange that only certian flags should be documented as producing poison. + + Cases of clear poison semantics not yet implemented: + - Exact flags on ashr/lshr produce poison + - NSW/NUW flags on shl produce poison + - Inbounds flag on getelementptr produce poison + - fptosi/fptoui (out of bounds input) produce poison + - Scalable vector types for insertelement/extractelement + - Floating point binary ops w/fmf nnan/noinfs flags produce poison + */ diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index 0ba8d5765e8c2..ca0cb4bdbe844 100644 --- a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -1,9 +1,8 @@ //===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,7 +61,10 @@ static const char *const SanCovTraceDiv4 = "__sanitizer_cov_trace_div4"; static const char *const SanCovTraceDiv8 = "__sanitizer_cov_trace_div8"; static const char *const SanCovTraceGep = "__sanitizer_cov_trace_gep"; static const char *const SanCovTraceSwitchName = "__sanitizer_cov_trace_switch"; -static const char *const SanCovModuleCtorName = "sancov.module_ctor"; +static const char *const SanCovModuleCtorTracePcGuardName = + "sancov.module_ctor_trace_pc_guard"; +static const char *const SanCovModuleCtor8bitCountersName = + "sancov.module_ctor_8bit_counters"; static const uint64_t SanCtorAndDtorPriority = 2; static const char *const SanCovTracePCGuardName = @@ -210,8 +212,9 @@ private: void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks); void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx, bool IsLeafFunc = true); - Function *CreateInitCallsForSections(Module &M, const char *InitFunctionName, - Type *Ty, const char *Section); + Function *CreateInitCallsForSections(Module &M, const char *CtorName, + const char *InitFunctionName, Type *Ty, + const char *Section); std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section, Type *Ty); @@ -223,13 +226,13 @@ private: std::string getSectionName(const std::string &Section) const; std::string getSectionStart(const std::string &Section) const; std::string getSectionEnd(const std::string &Section) const; - Function *SanCovTracePCIndir; - Function *SanCovTracePC, *SanCovTracePCGuard; - Function *SanCovTraceCmpFunction[4]; - Function *SanCovTraceConstCmpFunction[4]; - Function *SanCovTraceDivFunction[2]; - Function *SanCovTraceGepFunction; - Function *SanCovTraceSwitchFunction; + FunctionCallee SanCovTracePCIndir; + FunctionCallee SanCovTracePC, SanCovTracePCGuard; + FunctionCallee SanCovTraceCmpFunction[4]; + FunctionCallee SanCovTraceConstCmpFunction[4]; + FunctionCallee SanCovTraceDivFunction[2]; + FunctionCallee SanCovTraceGepFunction; + FunctionCallee SanCovTraceSwitchFunction; GlobalVariable *SanCovLowestStack; InlineAsm *EmptyAsm; Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy, @@ -270,24 +273,25 @@ SanitizerCoverageModule::CreateSecStartEnd(Module &M, const char *Section, // Account for the fact that on windows-msvc __start_* symbols actually // point to a uint64_t before the start of the array. auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy); - auto GEP = IRB.CreateGEP(SecStartI8Ptr, + auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr, ConstantInt::get(IntptrTy, sizeof(uint64_t))); return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEndPtr); } Function *SanitizerCoverageModule::CreateInitCallsForSections( - Module &M, const char *InitFunctionName, Type *Ty, + Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty, const char *Section) { auto SecStartEnd = CreateSecStartEnd(M, Section, Ty); auto SecStart = SecStartEnd.first; auto SecEnd = SecStartEnd.second; Function *CtorFunc; std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions( - M, SanCovModuleCtorName, InitFunctionName, {Ty, Ty}, {SecStart, SecEnd}); + M, CtorName, InitFunctionName, {Ty, Ty}, {SecStart, SecEnd}); + assert(CtorFunc->getName() == CtorName); if (TargetTriple.supportsCOMDAT()) { // Use comdat to dedup CtorFunc. - CtorFunc->setComdat(M.getOrInsertComdat(SanCovModuleCtorName)); + CtorFunc->setComdat(M.getOrInsertComdat(CtorName)); appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc); } else { appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority); @@ -329,77 +333,74 @@ bool SanitizerCoverageModule::runOnModule(Module &M) { Int16Ty = IRB.getInt16Ty(); Int8Ty = IRB.getInt8Ty(); - SanCovTracePCIndir = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy)); + SanCovTracePCIndir = + M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy); + // Make sure smaller parameters are zero-extended to i64 as required by the + // x86_64 ABI. + AttributeList SanCovTraceCmpZeroExtAL; + if (TargetTriple.getArch() == Triple::x86_64) { + SanCovTraceCmpZeroExtAL = + SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt); + SanCovTraceCmpZeroExtAL = + SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt); + } + SanCovTraceCmpFunction[0] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceCmp1, VoidTy, IRB.getInt8Ty(), IRB.getInt8Ty())); - SanCovTraceCmpFunction[1] = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(SanCovTraceCmp2, VoidTy, IRB.getInt16Ty(), - IRB.getInt16Ty())); - SanCovTraceCmpFunction[2] = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(SanCovTraceCmp4, VoidTy, IRB.getInt32Ty(), - IRB.getInt32Ty())); + M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy, + IRB.getInt8Ty(), IRB.getInt8Ty()); + SanCovTraceCmpFunction[1] = + M.getOrInsertFunction(SanCovTraceCmp2, SanCovTraceCmpZeroExtAL, VoidTy, + IRB.getInt16Ty(), IRB.getInt16Ty()); + SanCovTraceCmpFunction[2] = + M.getOrInsertFunction(SanCovTraceCmp4, SanCovTraceCmpZeroExtAL, VoidTy, + IRB.getInt32Ty(), IRB.getInt32Ty()); SanCovTraceCmpFunction[3] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty)); + M.getOrInsertFunction(SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty); - SanCovTraceConstCmpFunction[0] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceConstCmp1, VoidTy, Int8Ty, Int8Ty)); - SanCovTraceConstCmpFunction[1] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceConstCmp2, VoidTy, Int16Ty, Int16Ty)); - SanCovTraceConstCmpFunction[2] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceConstCmp4, VoidTy, Int32Ty, Int32Ty)); + SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction( + SanCovTraceConstCmp1, SanCovTraceCmpZeroExtAL, VoidTy, Int8Ty, Int8Ty); + SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction( + SanCovTraceConstCmp2, SanCovTraceCmpZeroExtAL, VoidTy, Int16Ty, Int16Ty); + SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction( + SanCovTraceConstCmp4, SanCovTraceCmpZeroExtAL, VoidTy, Int32Ty, Int32Ty); SanCovTraceConstCmpFunction[3] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty)); + M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty); - SanCovTraceDivFunction[0] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceDiv4, VoidTy, IRB.getInt32Ty())); + { + AttributeList AL; + if (TargetTriple.getArch() == Triple::x86_64) + AL = AL.addParamAttribute(*C, 0, Attribute::ZExt); + SanCovTraceDivFunction[0] = + M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty()); + } SanCovTraceDivFunction[1] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceDiv8, VoidTy, Int64Ty)); + M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty); SanCovTraceGepFunction = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceGep, VoidTy, IntptrTy)); + M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy); SanCovTraceSwitchFunction = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy)); + M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy); Constant *SanCovLowestStackConstant = M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy); - SanCovLowestStack = cast<GlobalVariable>(SanCovLowestStackConstant); + SanCovLowestStack = dyn_cast<GlobalVariable>(SanCovLowestStackConstant); + if (!SanCovLowestStack) { + C->emitError(StringRef("'") + SanCovLowestStackName + + "' should not be declared by the user"); + return true; + } SanCovLowestStack->setThreadLocalMode( GlobalValue::ThreadLocalMode::InitialExecTLSModel); if (Options.StackDepth && !SanCovLowestStack->isDeclaration()) SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy)); - // Make sure smaller parameters are zero-extended to i64 as required by the - // x86_64 ABI. - if (TargetTriple.getArch() == Triple::x86_64) { - for (int i = 0; i < 3; i++) { - SanCovTraceCmpFunction[i]->addParamAttr(0, Attribute::ZExt); - SanCovTraceCmpFunction[i]->addParamAttr(1, Attribute::ZExt); - SanCovTraceConstCmpFunction[i]->addParamAttr(0, Attribute::ZExt); - SanCovTraceConstCmpFunction[i]->addParamAttr(1, Attribute::ZExt); - } - SanCovTraceDivFunction[0]->addParamAttr(0, Attribute::ZExt); - } - - // We insert an empty inline asm after cov callbacks to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), StringRef(""), StringRef(""), /*hasSideEffects=*/true); - SanCovTracePC = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(SanCovTracePCName, VoidTy)); - SanCovTracePCGuard = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - SanCovTracePCGuardName, VoidTy, Int32PtrTy)); + SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy); + SanCovTracePCGuard = + M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, Int32PtrTy); for (auto &F : M) runOnFunction(F); @@ -407,14 +408,16 @@ bool SanitizerCoverageModule::runOnModule(Module &M) { Function *Ctor = nullptr; if (FunctionGuardArray) - Ctor = CreateInitCallsForSections(M, SanCovTracePCGuardInitName, Int32PtrTy, + Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName, + SanCovTracePCGuardInitName, Int32PtrTy, SanCovGuardsSectionName); if (Function8bitCounterArray) - Ctor = CreateInitCallsForSections(M, SanCov8bitCountersInitName, Int8PtrTy, + Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName, + SanCov8bitCountersInitName, Int8PtrTy, SanCovCountersSectionName); if (Ctor && Options.PCTable) { auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrPtrTy); - Function *InitFunction = declareSanitizerInitFunction( + FunctionCallee InitFunction = declareSanitizerInitFunction( M, SanCovPCsInitName, {IntptrPtrTy, IntptrPtrTy}); IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator()); IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second}); @@ -458,12 +461,12 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, const DominatorTree *DT, const PostDominatorTree *PDT, const SanitizerCoverageOptions &Options) { - // Don't insert coverage for unreachable blocks: we will never call - // __sanitizer_cov() for them, so counting them in + // Don't insert coverage for blocks containing nothing but unreachable: we + // will never call __sanitizer_cov() for them, so counting them in // NumberOfInstrumentedBlocks() might complicate calculation of code coverage // percentage. Also, unreachable instructions frequently have no debug // locations. - if (isa<UnreachableInst>(BB->getTerminator())) + if (isa<UnreachableInst>(BB->getFirstNonPHIOrDbgOrLifetime())) return false; // Don't insert coverage into blocks without a valid insertion point @@ -484,6 +487,37 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, && !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor()); } + +// Returns true iff From->To is a backedge. +// A twist here is that we treat From->To as a backedge if +// * To dominates From or +// * To->UniqueSuccessor dominates From +static bool IsBackEdge(BasicBlock *From, BasicBlock *To, + const DominatorTree *DT) { + if (DT->dominates(To, From)) + return true; + if (auto Next = To->getUniqueSuccessor()) + if (DT->dominates(Next, From)) + return true; + return false; +} + +// Prunes uninteresting Cmp instrumentation: +// * CMP instructions that feed into loop backedge branch. +// +// Note that Cmp pruning is controlled by the same flag as the +// BB pruning. +static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, + const SanitizerCoverageOptions &Options) { + if (!Options.NoPrune) + if (CMP->hasOneUse()) + if (auto BR = dyn_cast<BranchInst>(CMP->user_back())) + for (BasicBlock *B : BR->successors()) + if (IsBackEdge(BR->getParent(), B, DT)) + return false; + return true; +} + bool SanitizerCoverageModule::runOnFunction(Function &F) { if (F.empty()) return false; @@ -508,7 +542,7 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) { isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) return false; if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) - SplitAllCriticalEdges(F); + SplitAllCriticalEdges(F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests()); SmallVector<Instruction *, 8> IndirCalls; SmallVector<BasicBlock *, 16> BlocksToInstrument; SmallVector<Instruction *, 8> CmpTraceTargets; @@ -532,8 +566,9 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) { IndirCalls.push_back(&Inst); } if (Options.TraceCmp) { - if (isa<ICmpInst>(&Inst)) - CmpTraceTargets.push_back(&Inst); + if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst)) + if (IsInterestingCmp(CMP, DT, Options)) + CmpTraceTargets.push_back(&Inst); if (isa<SwitchInst>(&Inst)) SwitchTraceTargets.push_back(&Inst); } @@ -797,9 +832,9 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } if (Options.Inline8bitCounters) { auto CounterPtr = IRB.CreateGEP( - Function8bitCounterArray, + Function8bitCounterArray->getValueType(), Function8bitCounterArray, {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); - auto Load = IRB.CreateLoad(CounterPtr); + auto Load = IRB.CreateLoad(Int8Ty, CounterPtr); auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1)); auto Store = IRB.CreateStore(Inc, CounterPtr); SetNoSanitizeMetadata(Load); @@ -812,7 +847,7 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB, auto FrameAddrPtr = IRB.CreateCall(GetFrameAddr, {Constant::getNullValue(Int32Ty)}); auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy); - auto LowestStack = IRB.CreateLoad(SanCovLowestStack); + auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack); auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack); auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false); IRBuilder<> ThenIRB(ThenTerm); diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index 077364e15c4fa..5be13fa745cbd 100644 --- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -1,9 +1,8 @@ //===-- ThreadSanitizer.cpp - race detector -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -111,25 +110,26 @@ private: Type *IntptrTy; IntegerType *OrdTy; // Callbacks to run-time library are computed in doInitialization. - Function *TsanFuncEntry; - Function *TsanFuncExit; - Function *TsanIgnoreBegin; - Function *TsanIgnoreEnd; + FunctionCallee TsanFuncEntry; + FunctionCallee TsanFuncExit; + FunctionCallee TsanIgnoreBegin; + FunctionCallee TsanIgnoreEnd; // Accesses sizes are powers of two: 1, 2, 4, 8, 16. static const size_t kNumberOfAccessSizes = 5; - Function *TsanRead[kNumberOfAccessSizes]; - Function *TsanWrite[kNumberOfAccessSizes]; - Function *TsanUnalignedRead[kNumberOfAccessSizes]; - Function *TsanUnalignedWrite[kNumberOfAccessSizes]; - Function *TsanAtomicLoad[kNumberOfAccessSizes]; - Function *TsanAtomicStore[kNumberOfAccessSizes]; - Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; - Function *TsanAtomicCAS[kNumberOfAccessSizes]; - Function *TsanAtomicThreadFence; - Function *TsanAtomicSignalFence; - Function *TsanVptrUpdate; - Function *TsanVptrLoad; - Function *MemmoveFn, *MemcpyFn, *MemsetFn; + FunctionCallee TsanRead[kNumberOfAccessSizes]; + FunctionCallee TsanWrite[kNumberOfAccessSizes]; + FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes]; + FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]; + FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes]; + FunctionCallee TsanAtomicStore[kNumberOfAccessSizes]; + FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1] + [kNumberOfAccessSizes]; + FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes]; + FunctionCallee TsanAtomicThreadFence; + FunctionCallee TsanAtomicSignalFence; + FunctionCallee TsanVptrUpdate; + FunctionCallee TsanVptrLoad; + FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; Function *TsanCtorFunction; }; @@ -189,14 +189,14 @@ void ThreadSanitizer::initializeCallbacks(Module &M) { Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex, Attribute::NoUnwind); // Initialize the callbacks. - TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_func_entry", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); - TsanFuncExit = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy())); - TsanIgnoreBegin = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_ignore_thread_begin", Attr, IRB.getVoidTy())); - TsanIgnoreEnd = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_ignore_thread_end", Attr, IRB.getVoidTy())); + TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr, + IRB.getVoidTy(), IRB.getInt8PtrTy()); + TsanFuncExit = + M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()); + TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr, + IRB.getVoidTy()); + TsanIgnoreEnd = + M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy()); OrdTy = IRB.getInt32Ty(); for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { const unsigned ByteSize = 1U << i; @@ -204,32 +204,30 @@ void ThreadSanitizer::initializeCallbacks(Module &M) { std::string ByteSizeStr = utostr(ByteSize); std::string BitSizeStr = utostr(BitSize); SmallString<32> ReadName("__tsan_read" + ByteSizeStr); - TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - ReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); + TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), + IRB.getInt8PtrTy()); SmallString<32> WriteName("__tsan_write" + ByteSizeStr); - TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - WriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); + TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), + IRB.getInt8PtrTy()); SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); - TsanUnalignedRead[i] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); + TsanUnalignedRead[i] = M.getOrInsertFunction( + UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); - TsanUnalignedWrite[i] = - checkSanitizerInterfaceFunction(M.getOrInsertFunction( - UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); + TsanUnalignedWrite[i] = M.getOrInsertFunction( + UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); Type *Ty = Type::getIntNTy(M.getContext(), BitSize); Type *PtrTy = Ty->getPointerTo(); SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); - TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy)); + TsanAtomicLoad[i] = + M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy); SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); - TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy)); + TsanAtomicStore[i] = M.getOrInsertFunction( + AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy); for (int op = AtomicRMWInst::FIRST_BINOP; op <= AtomicRMWInst::LAST_BINOP; ++op) { @@ -252,34 +250,34 @@ void ThreadSanitizer::initializeCallbacks(Module &M) { else continue; SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); - TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( - M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy)); + TsanAtomicRMW[op][i] = + M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy); } SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + "_compare_exchange_val"); - TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - AtomicCASName, Attr, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy)); + TsanAtomicCAS[i] = M.getOrInsertFunction(AtomicCASName, Attr, Ty, PtrTy, Ty, + Ty, OrdTy, OrdTy); } - TsanVptrUpdate = checkSanitizerInterfaceFunction( + TsanVptrUpdate = M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), - IRB.getInt8PtrTy(), IRB.getInt8PtrTy())); - TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_vptr_read", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy())); - TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_atomic_thread_fence", Attr, IRB.getVoidTy(), OrdTy)); - TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( - "__tsan_atomic_signal_fence", Attr, IRB.getVoidTy(), OrdTy)); + IRB.getInt8PtrTy(), IRB.getInt8PtrTy()); + TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr, + IRB.getVoidTy(), IRB.getInt8PtrTy()); + TsanAtomicThreadFence = M.getOrInsertFunction("__tsan_atomic_thread_fence", + Attr, IRB.getVoidTy(), OrdTy); + TsanAtomicSignalFence = M.getOrInsertFunction("__tsan_atomic_signal_fence", + Attr, IRB.getVoidTy(), OrdTy); - MemmoveFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IntptrTy)); - MemcpyFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt8PtrTy(), IntptrTy)); - MemsetFn = checkSanitizerInterfaceFunction( - M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), - IRB.getInt32Ty(), IntptrTy)); + MemmoveFn = + M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); + MemcpyFn = + M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); + MemsetFn = + M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), + IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy); } ThreadSanitizer::ThreadSanitizer(Module &M) { @@ -291,7 +289,9 @@ ThreadSanitizer::ThreadSanitizer(Module &M) { /*InitArgs=*/{}, // This callback is invoked when the functions are created the first // time. Hook them into the global ctors list in that case: - [&](Function *Ctor, Function *) { appendToGlobalCtors(M, Ctor, 0); }); + [&](Function *Ctor, FunctionCallee) { + appendToGlobalCtors(M, Ctor, 0); + }); } static bool isVtableAccess(Instruction *I) { @@ -559,7 +559,7 @@ bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, : cast<LoadInst>(I)->getAlignment(); Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); - Value *OnAccessFunc = nullptr; + FunctionCallee OnAccessFunc = nullptr; if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; else @@ -659,7 +659,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { int Idx = getMemoryAccessFuncIndex(Addr, DL); if (Idx < 0) return false; - Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; + FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx]; if (!F) return false; const unsigned ByteSize = 1U << Idx; @@ -706,8 +706,9 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { I->eraseFromParent(); } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; - Function *F = FI->getSyncScopeID() == SyncScope::SingleThread ? - TsanAtomicSignalFence : TsanAtomicThreadFence; + FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread + ? TsanAtomicSignalFence + : TsanAtomicThreadFence; CallInst *C = CallInst::Create(F, Args); ReplaceInstWithInst(I, C); } |