diff options
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation')
15 files changed, 180 insertions, 163 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index d92ee11c2e1a..79c119489a65 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -59,6 +59,7 @@ #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" @@ -1808,6 +1809,8 @@ bool ModuleAddressSanitizer::ShouldInstrumentGlobal(GlobalVariable *G) { if (GlobalsMD.get(G).IsBlacklisted) return false; if (!Ty->isSized()) return false; if (!G->hasInitializer()) return false; + // Only instrument globals of default address spaces + if (G->getAddressSpace()) return false; if (GlobalWasGeneratedByCompiler(G)) return false; // Our own globals. // Two problems with thread-locals: // - The address of the main thread's copy can't be computed at link-time. @@ -2898,15 +2901,14 @@ void FunctionStackPoisoner::copyArgsPassedByValToAllocas() { for (Argument &Arg : F.args()) { if (Arg.hasByValAttr()) { Type *Ty = Arg.getType()->getPointerElementType(); - unsigned Alignment = Arg.getParamAlignment(); - if (Alignment == 0) - Alignment = DL.getABITypeAlignment(Ty); + const Align Alignment = + DL.getValueOrABITypeAlignment(Arg.getParamAlign(), Ty); AllocaInst *AI = IRB.CreateAlloca( Ty, nullptr, (Arg.hasName() ? Arg.getName() : "Arg" + Twine(Arg.getArgNo())) + ".byval"); - AI->setAlignment(Align(Alignment)); + AI->setAlignment(Alignment); Arg.replaceAllUsesWith(AI); uint64_t AllocSize = DL.getTypeAllocSize(Ty); @@ -2993,7 +2995,6 @@ void FunctionStackPoisoner::processStaticAllocas() { Instruction *InsBefore = AllocaVec[0]; IRBuilder<> IRB(InsBefore); - IRB.SetCurrentDebugLocation(EntryDebugLocation); // Make sure non-instrumented allocas stay in the entry block. Otherwise, // debug info is broken, because only entry-block allocas are treated as @@ -3088,14 +3089,12 @@ void FunctionStackPoisoner::processStaticAllocas() { Instruction *Term = SplitBlockAndInsertIfThen(UseAfterReturnIsEnabled, InsBefore, false); IRBuilder<> IRBIf(Term); - IRBIf.SetCurrentDebugLocation(EntryDebugLocation); StackMallocIdx = StackMallocSizeClass(LocalStackSize); assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass); Value *FakeStackValue = IRBIf.CreateCall(AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); IRB.SetInsertPoint(InsBefore); - IRB.SetCurrentDebugLocation(EntryDebugLocation); FakeStack = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, Term, ConstantInt::get(IntptrTy, 0)); @@ -3103,14 +3102,11 @@ void FunctionStackPoisoner::processStaticAllocas() { IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy)); Term = SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false); IRBIf.SetInsertPoint(Term); - IRBIf.SetCurrentDebugLocation(EntryDebugLocation); Value *AllocaValue = DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca; IRB.SetInsertPoint(InsBefore); - IRB.SetCurrentDebugLocation(EntryDebugLocation); LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack); - IRB.SetCurrentDebugLocation(EntryDebugLocation); IRB.CreateStore(LocalStackBase, LocalStackBaseAlloca); DIExprFlags |= DIExpression::DerefBefore; } else { diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp index ae34be986537..9abb62ac788c 100644 --- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp +++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp @@ -24,6 +24,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp index 55c64fa4b727..d35abb92dd08 100644 --- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp +++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp @@ -27,7 +27,9 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/BranchProbability.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -623,6 +625,10 @@ static bool checkMDProf(MDNode *MD, BranchProbability &TrueProb, assert(SumWt >= TrueWt && SumWt >= FalseWt && "Overflow calculating branch probabilities."); + // Guard against 0-to-0 branch weights to avoid a division-by-zero crash. + if (SumWt == 0) + return false; + TrueProb = BranchProbability::getBranchProbability(TrueWt, SumWt); FalseProb = BranchProbability::getBranchProbability(FalseWt, SumWt); return true; @@ -1061,6 +1067,7 @@ static bool shouldSplit(Instruction *InsertPoint, DenseSet<Value *> &ConditionValues, DominatorTree &DT, DenseSet<Instruction *> &Unhoistables) { + assert(InsertPoint && "Null InsertPoint"); CHR_DEBUG( dbgs() << "shouldSplit " << *InsertPoint << " PrevConditionValues "; for (Value *V : PrevConditionValues) { @@ -1071,7 +1078,6 @@ static bool shouldSplit(Instruction *InsertPoint, dbgs() << *V << ", "; } dbgs() << "\n"); - assert(InsertPoint && "Null InsertPoint"); // If any of Bases isn't hoistable to the hoist point, split. for (Value *V : ConditionValues) { DenseMap<Instruction *, bool> Visited; diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index c0353cba0b2f..cf9a6a321c7a 100644 --- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -55,7 +55,6 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" @@ -83,13 +82,16 @@ #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SpecialCaseList.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -479,7 +481,9 @@ DataFlowSanitizer::DataFlowSanitizer( std::vector<std::string> AllABIListFiles(std::move(ABIListFiles)); AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(), ClABIListFiles.end()); - ABIList.set(SpecialCaseList::createOrDie(AllABIListFiles)); + // FIXME: should we propagate vfs::FileSystem to this constructor? + ABIList.set( + SpecialCaseList::createOrDie(AllABIListFiles, *vfs::getRealFileSystem())); } FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) { diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp index ac6082441eae..bf3e4ed3e31f 100644 --- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -30,6 +30,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -128,9 +129,9 @@ private: // Checksum, produced by hash of EdgeDestinations SmallVector<uint32_t, 4> FileChecksums; - Module *M; + Module *M = nullptr; std::function<const TargetLibraryInfo &(Function &F)> GetTLI; - LLVMContext *Ctx; + LLVMContext *Ctx = nullptr; SmallVector<std::unique_ptr<GCOVFunction>, 16> Funcs; std::vector<Regex> FilterRe; std::vector<Regex> ExcludeRe; @@ -384,7 +385,7 @@ namespace { return EdgeDestinations; } - uint32_t getFuncChecksum() { + uint32_t getFuncChecksum() const { return FuncChecksum; } @@ -713,7 +714,10 @@ void GCOVProfiler::emitProfileNotes() { // to have a counter for the function definition. uint32_t Line = SP->getLine(); auto Filename = getFilename(SP); - Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line); + + // Artificial functions such as global initializers + if (!SP->isArtificial()) + Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line); for (auto &BB : F) { GCOVBlock &Block = Func.getBlock(&BB); diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index f87132ee4758..7e8f8e27a97b 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -38,6 +38,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" @@ -221,7 +222,7 @@ public: Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong); bool instrumentStack( SmallVectorImpl<AllocaInst *> &Allocas, - DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap, + DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap, SmallVectorImpl<Instruction *> &RetVec, Value *StackTag); Value *readRegister(IRBuilder<> &IRB, StringRef Name); bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec); @@ -284,7 +285,6 @@ private: FunctionCallee HwasanTagMemoryFunc; FunctionCallee HwasanGenerateTagFunc; - FunctionCallee HwasanThreadEnterFunc; Constant *ShadowGlobal; @@ -473,9 +473,6 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) { HWAsanHandleVfork = M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy); - - HwasanThreadEnterFunc = - M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy()); } Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) { @@ -792,7 +789,7 @@ bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, // llvm.memset right here into either a sequence of stores, or a call to // hwasan_tag_memory. if (ShadowSize) - IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1); + IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align::None()); if (Size != AlignedSize) { IRB.CreateStore( ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()), @@ -934,34 +931,13 @@ void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); assert(SlotPtr); - Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); - - Function *F = IRB.GetInsertBlock()->getParent(); - if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") { - Value *ThreadLongEqZero = - IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0)); - auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen( - ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(), - false, MDBuilder(*C).createBranchWeights(1, 100000))); - - IRB.SetInsertPoint(Br); - // FIXME: This should call a new runtime function with a custom calling - // convention to avoid needing to spill all arguments here. - IRB.CreateCall(HwasanThreadEnterFunc); - LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); - - IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin()); - PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2); - ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent()); - ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent()); - ThreadLong = ThreadLongPhi; - } - + Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI. Value *ThreadLongMaybeUntagged = TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); if (WithFrameRecord) { + Function *F = IRB.GetInsertBlock()->getParent(); StackBaseTag = IRB.CreateAShr(ThreadLong, 3); // Prepare ring buffer data. @@ -1040,7 +1016,7 @@ bool HWAddressSanitizer::instrumentLandingPads( bool HWAddressSanitizer::instrumentStack( SmallVectorImpl<AllocaInst *> &Allocas, - DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap, + DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap, SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) { // Ideally, we want to calculate tagged stack base pointer, and rewrite all // alloca addresses using that. Unfortunately, offsets are not known yet @@ -1062,11 +1038,15 @@ bool HWAddressSanitizer::instrumentStack( AI->replaceUsesWithIf(Replacement, [AILong](Use &U) { return U.getUser() != AILong; }); - for (auto *DDI : AllocaDeclareMap.lookup(AI)) { - DIExpression *OldExpr = DDI->getExpression(); - DIExpression *NewExpr = DIExpression::append( - OldExpr, {dwarf::DW_OP_LLVM_tag_offset, RetagMask(N)}); - DDI->setArgOperand(2, MetadataAsValue::get(*C, NewExpr)); + for (auto *DDI : AllocaDbgMap.lookup(AI)) { + // Prepend "tag_offset, N" to the dwarf expression. + // Tag offset logically applies to the alloca pointer, and it makes sense + // to put it at the beginning of the expression. + SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset, + RetagMask(N)}; + DDI->setArgOperand( + 2, MetadataAsValue::get(*C, DIExpression::prependOpcodes( + DDI->getExpression(), NewOps))); } size_t Size = getAllocaSizeInBytes(*AI); @@ -1113,7 +1093,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) { SmallVector<AllocaInst*, 8> AllocasToInstrument; SmallVector<Instruction*, 8> RetVec; SmallVector<Instruction*, 8> LandingPadVec; - DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> AllocaDeclareMap; + DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap; for (auto &BB : F) { for (auto &Inst : BB) { if (ClInstrumentStack) @@ -1127,9 +1107,10 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) { isa<CleanupReturnInst>(Inst)) RetVec.push_back(&Inst); - if (auto *DDI = dyn_cast<DbgDeclareInst>(&Inst)) - if (auto *Alloca = dyn_cast_or_null<AllocaInst>(DDI->getAddress())) - AllocaDeclareMap[Alloca].push_back(DDI); + if (auto *DDI = dyn_cast<DbgVariableIntrinsic>(&Inst)) + if (auto *Alloca = + dyn_cast_or_null<AllocaInst>(DDI->getVariableLocation())) + AllocaDbgMap[Alloca].push_back(DDI); if (InstrumentLandingPads && isa<LandingPadInst>(Inst)) LandingPadVec.push_back(&Inst); @@ -1172,7 +1153,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) { if (!AllocasToInstrument.empty()) { Value *StackTag = ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB); - Changed |= instrumentStack(AllocasToInstrument, AllocaDeclareMap, RetVec, + Changed |= instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, StackTag); } diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 74d6e76eceb6..d5787c8f62a1 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -36,6 +36,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Casting.h" diff --git a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp index 93d3a8a14d5c..518b8895e836 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp @@ -9,6 +9,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/InstrOrderFile.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" @@ -19,6 +20,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/PassRegistry.h" #include "llvm/ProfileData/InstrProf.h" @@ -28,7 +30,6 @@ #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Instrumentation/InstrOrderFile.h" #include <fstream> #include <map> #include <mutex> diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 1f092a5f3103..04c7e856b5d4 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -37,6 +37,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Casting.h" diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 69c9020e060b..80acab307578 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -170,12 +170,14 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsX86.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/IR/ValueMap.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Casting.h" @@ -202,8 +204,8 @@ using namespace llvm; #define DEBUG_TYPE "msan" static const unsigned kOriginSize = 4; -static const unsigned kMinOriginAlignment = 4; -static const unsigned kShadowTLSAlignment = 8; +static const Align kMinOriginAlignment = Align(4); +static const Align kShadowTLSAlignment = Align(8); // These constants must be kept in sync with the ones in msan.h. static const unsigned kParamTLSSize = 800; @@ -1086,15 +1088,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { /// Fill memory range with the given origin value. void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr, - unsigned Size, unsigned Alignment) { + unsigned Size, Align Alignment) { const DataLayout &DL = F.getParent()->getDataLayout(); - unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy); + const Align IntptrAlignment = Align(DL.getABITypeAlignment(MS.IntptrTy)); unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy); assert(IntptrAlignment >= kMinOriginAlignment); assert(IntptrSize >= kOriginSize); unsigned Ofs = 0; - unsigned CurrentAlignment = Alignment; + Align CurrentAlignment = Alignment; if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) { Value *IntptrOrigin = originToIntptr(IRB, Origin); Value *IntptrOriginPtr = @@ -1102,7 +1104,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { for (unsigned i = 0; i < Size / IntptrSize; ++i) { Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i) : IntptrOriginPtr; - IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment); + IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment.value()); Ofs += IntptrSize / kOriginSize; CurrentAlignment = IntptrAlignment; } @@ -1111,23 +1113,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) { Value *GEP = i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr; - IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment); + IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment.value()); CurrentAlignment = kMinOriginAlignment; } } void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin, - Value *OriginPtr, unsigned Alignment, bool AsCall) { + Value *OriginPtr, Align Alignment, bool AsCall) { const DataLayout &DL = F.getParent()->getDataLayout(); - unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); + const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); if (Shadow->getType()->isAggregateType()) { paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize, OriginAlignment); } else { Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); - Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow); - if (ConstantShadow) { + if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) { if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize, OriginAlignment); @@ -1164,12 +1165,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val); Value *ShadowPtr, *OriginPtr; Type *ShadowTy = Shadow->getType(); - unsigned Alignment = SI->getAlignment(); - unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); + const Align Alignment = assumeAligned(SI->getAlignment()); + const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true); - StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment); + StoreInst *NewSI = + IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment.value()); LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n"); (void)NewSI; @@ -1207,8 +1209,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n"); - Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow); - if (ConstantShadow) { + if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) { if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) { insertWarningFn(IRB, Origin); } @@ -1403,10 +1404,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { /// /// Shadow = ShadowBase + Offset /// Origin = (OriginBase + Offset) & ~3ULL - std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr, - IRBuilder<> &IRB, - Type *ShadowTy, - unsigned Alignment) { + std::pair<Value *, Value *> + getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, + MaybeAlign Alignment) { Value *ShadowOffset = getShadowPtrOffset(Addr, IRB); Value *ShadowLong = ShadowOffset; uint64_t ShadowBase = MS.MapParams->ShadowBase; @@ -1424,8 +1424,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (OriginBase != 0) OriginLong = IRB.CreateAdd(OriginLong, ConstantInt::get(MS.IntptrTy, OriginBase)); - if (Alignment < kMinOriginAlignment) { - uint64_t Mask = kMinOriginAlignment - 1; + if (!Alignment || *Alignment < kMinOriginAlignment) { + uint64_t Mask = kMinOriginAlignment.value() - 1; OriginLong = IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask)); } @@ -1435,9 +1435,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { return std::make_pair(ShadowPtr, OriginPtr); } - std::pair<Value *, Value *> - getShadowOriginPtrKernel(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, - unsigned Alignment, bool isStore) { + std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr, + IRBuilder<> &IRB, + Type *ShadowTy, + bool isStore) { Value *ShadowOriginPtrs; const DataLayout &DL = F.getParent()->getDataLayout(); int Size = DL.getTypeStoreSize(ShadowTy); @@ -1462,14 +1463,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, - unsigned Alignment, + MaybeAlign Alignment, bool isStore) { - std::pair<Value *, Value *> ret; if (MS.CompileKernel) - ret = getShadowOriginPtrKernel(Addr, IRB, ShadowTy, Alignment, isStore); - else - ret = getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); - return ret; + return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore); + return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); } /// Compute the shadow address for a given function argument. @@ -1619,11 +1617,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // ByVal pointer itself has clean shadow. We copy the actual // argument shadow to the underlying memory. // Figure out maximal valid memcpy alignment. - unsigned ArgAlign = FArg.getParamAlignment(); - if (ArgAlign == 0) { - Type *EltType = A->getType()->getPointerElementType(); - ArgAlign = DL.getABITypeAlignment(EltType); - } + const Align ArgAlign = DL.getValueOrABITypeAlignment( + MaybeAlign(FArg.getParamAlignment()), + A->getType()->getPointerElementType()); Value *CpShadowPtr = getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign, /*isStore*/ true) @@ -1635,7 +1631,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign); } else { - unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); + const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base, CopyAlign, Size); LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n"); @@ -1647,8 +1643,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // ParamTLS overflow. *ShadowPtr = getCleanShadow(V); } else { - *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base, - kShadowTLSAlignment); + *ShadowPtr = EntryIRB.CreateAlignedLoad( + getShadowTy(&FArg), Base, kShadowTLSAlignment.value()); } } LLVM_DEBUG(dbgs() @@ -1782,13 +1778,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRB(I.getNextNode()); Type *ShadowTy = getShadowTy(&I); Value *Addr = I.getPointerOperand(); - Value *ShadowPtr, *OriginPtr; - unsigned Alignment = I.getAlignment(); + Value *ShadowPtr = nullptr, *OriginPtr = nullptr; + const Align Alignment = assumeAligned(I.getAlignment()); if (PropagateShadow) { std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); - setShadow(&I, - IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); + setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, + Alignment.value(), "_msld")); } else { setShadow(&I, getCleanShadow(&I)); } @@ -1801,9 +1797,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (MS.TrackOrigins) { if (PropagateShadow) { - unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); - setOrigin( - &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment)); + const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); + setOrigin(&I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, + OriginAlignment.value())); } else { setOrigin(&I, getCleanOrigin()); } @@ -1825,8 +1821,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRB(&I); Value *Addr = I.getOperand(0); - Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(), - /*Alignment*/ 1, /*isStore*/ true) + Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(), Align::None(), + /*isStore*/ true) .first; if (ClCheckAccessAddress) @@ -2458,7 +2454,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // We don't know the pointer alignment (could be unaligned SSE store!). // Have to assume to worst case. std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr( - Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true); + Addr, IRB, Shadow->getType(), Align::None(), /*isStore*/ true); IRB.CreateAlignedStore(Shadow, ShadowPtr, 1); if (ClCheckAccessAddress) @@ -2478,15 +2474,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *Addr = I.getArgOperand(0); Type *ShadowTy = getShadowTy(&I); - Value *ShadowPtr, *OriginPtr; + Value *ShadowPtr = nullptr, *OriginPtr = nullptr; if (PropagateShadow) { // We don't know the pointer alignment (could be unaligned SSE load!). // Have to assume to worst case. - unsigned Alignment = 1; + const Align Alignment = Align::None(); std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); - setShadow(&I, - IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); + setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, + Alignment.value(), "_msld")); } else { setShadow(&I, getCleanShadow(&I)); } @@ -2873,7 +2869,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value* Addr = I.getArgOperand(0); Type *Ty = IRB.getInt32Ty(); Value *ShadowPtr = - getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true) + getShadowOriginPtr(Addr, IRB, Ty, Align::None(), /*isStore*/ true) .first; IRB.CreateStore(getCleanShadow(Ty), @@ -2889,7 +2885,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRB(&I); Value *Addr = I.getArgOperand(0); Type *Ty = IRB.getInt32Ty(); - unsigned Alignment = 1; + const Align Alignment = Align::None(); Value *ShadowPtr, *OriginPtr; std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false); @@ -2897,7 +2893,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (ClCheckAccessAddress) insertShadowCheck(Addr, &I); - Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr"); + Value *Shadow = + IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment.value(), "_ldmxcsr"); Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr) : getCleanOrigin(); insertShadowCheck(Shadow, Origin, &I); @@ -2907,14 +2904,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRB(&I); Value *V = I.getArgOperand(0); Value *Addr = I.getArgOperand(1); - unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); + const MaybeAlign Alignment( + cast<ConstantInt>(I.getArgOperand(2))->getZExtValue()); Value *Mask = I.getArgOperand(3); Value *Shadow = getShadow(V); Value *ShadowPtr; Value *OriginPtr; std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr( - Addr, IRB, Shadow->getType(), Align, /*isStore*/ true); + Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true); if (ClCheckAccessAddress) { insertShadowCheck(Addr, &I); @@ -2923,20 +2921,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { insertShadowCheck(Mask, &I); } - IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask); + IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment ? Alignment->value() : 0, + Mask); if (MS.TrackOrigins) { auto &DL = F.getParent()->getDataLayout(); paintOrigin(IRB, getOrigin(V), OriginPtr, DL.getTypeStoreSize(Shadow->getType()), - std::max(Align, kMinOriginAlignment)); + llvm::max(Alignment, kMinOriginAlignment)); } } bool handleMaskedLoad(IntrinsicInst &I) { IRBuilder<> IRB(&I); Value *Addr = I.getArgOperand(0); - unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); + const MaybeAlign Alignment( + cast<ConstantInt>(I.getArgOperand(1))->getZExtValue()); Value *Mask = I.getArgOperand(2); Value *PassThru = I.getArgOperand(3); @@ -2944,9 +2944,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Value *ShadowPtr, *OriginPtr; if (PropagateShadow) { std::tie(ShadowPtr, OriginPtr) = - getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false); - setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask, - getShadow(PassThru), "_msmaskedld")); + getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); + setShadow(&I, IRB.CreateMaskedLoad( + ShadowPtr, Alignment ? Alignment->value() : 0, Mask, + getShadow(PassThru), "_msmaskedld")); } else { setShadow(&I, getCleanShadow(&I)); } @@ -3278,7 +3279,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // Clear out readonly/readnone attributes. AttrBuilder B; B.addAttribute(Attribute::ReadOnly) - .addAttribute(Attribute::ReadNone); + .addAttribute(Attribute::ReadNone) + .addAttribute(Attribute::WriteOnly) + .addAttribute(Attribute::ArgMemOnly) + .addAttribute(Attribute::Speculatable); Func->removeAttributes(AttributeList::FunctionIndex, B); } @@ -3312,8 +3316,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { "ByVal argument is not a pointer!"); Size = DL.getTypeAllocSize(A->getType()->getPointerElementType()); if (ArgOffset + Size > kParamTLSSize) break; - unsigned ParamAlignment = CS.getParamAlignment(i); - unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment); + const MaybeAlign ParamAlignment(CS.getParamAlignment(i)); + MaybeAlign Alignment = llvm::None; + if (ParamAlignment) + Alignment = std::min(*ParamAlignment, kShadowTLSAlignment); Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ false) @@ -3326,7 +3332,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { Size = DL.getTypeAllocSize(A->getType()); if (ArgOffset + Size > kParamTLSSize) break; Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase, - kShadowTLSAlignment); + kShadowTLSAlignment.value()); Constant *Cst = dyn_cast<Constant>(ArgShadow); if (Cst && Cst->isNullValue()) ArgIsInitialized = true; } @@ -3352,7 +3358,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRBBefore(&I); // Until we have full dynamic coverage, make sure the retval shadow is 0. Value *Base = getShadowPtrForRetval(&I, IRBBefore); - IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment); + IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, + kShadowTLSAlignment.value()); BasicBlock::iterator NextInsn; if (CS.isCall()) { NextInsn = ++I.getIterator(); @@ -3376,7 +3383,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { IRBuilder<> IRBAfter(&*NextInsn); Value *RetvalShadow = IRBAfter.CreateAlignedLoad( getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter), - kShadowTLSAlignment, "_msret"); + kShadowTLSAlignment.value(), "_msret"); setShadow(&I, RetvalShadow); if (MS.TrackOrigins) setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy, @@ -3403,10 +3410,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { if (CheckReturnValue) { insertShadowCheck(RetVal, &I); Value *Shadow = getCleanShadow(RetVal); - IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); + IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value()); } else { Value *Shadow = getShadow(RetVal); - IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); + IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value()); if (MS.TrackOrigins) IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB)); } @@ -3447,11 +3454,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); } else { Value *ShadowBase, *OriginBase; - std::tie(ShadowBase, OriginBase) = - getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), 1, /*isStore*/ true); + std::tie(ShadowBase, OriginBase) = getShadowOriginPtr( + &I, IRB, IRB.getInt8Ty(), Align::None(), /*isStore*/ true); Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); - IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment()); + IRB.CreateMemSet(ShadowBase, PoisonValue, Len, + MaybeAlign(I.getAlignment())); } if (PoisonStack && MS.TrackOrigins) { @@ -3863,7 +3871,7 @@ struct VarArgAMD64Helper : public VarArgHelper { if (!ShadowBase) continue; Value *Shadow = MSV.getShadow(A); - IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment); + IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment.value()); if (MS.TrackOrigins) { Value *Origin = MSV.getOrigin(A); unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); @@ -3904,7 +3912,7 @@ struct VarArgAMD64Helper : public VarArgHelper { IRBuilder<> IRB(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); @@ -3942,10 +3950,11 @@ struct VarArgAMD64Helper : public VarArgHelper { IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); if (MS.TrackOrigins) { VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSOriginCopy, 8, MS.VAArgOriginTLS, 8, CopySize); + IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS, + Align(8), CopySize); } } @@ -3964,7 +3973,7 @@ struct VarArgAMD64Helper : public VarArgHelper { Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; - unsigned Alignment = 16; + const Align Alignment = Align(16); std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); @@ -4032,7 +4041,8 @@ struct VarArgMIPS64Helper : public VarArgHelper { VAArgOffset = alignTo(VAArgOffset, 8); if (!Base) continue; - IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); + IRB.CreateAlignedStore(MSV.getShadow(A), Base, + kShadowTLSAlignment.value()); } Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset); @@ -4058,7 +4068,7 @@ struct VarArgMIPS64Helper : public VarArgHelper { VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), @@ -4070,7 +4080,7 @@ struct VarArgMIPS64Helper : public VarArgHelper { VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), @@ -4089,7 +4099,7 @@ struct VarArgMIPS64Helper : public VarArgHelper { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); } // Instrument va_start. @@ -4105,7 +4115,7 @@ struct VarArgMIPS64Helper : public VarArgHelper { Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); @@ -4203,7 +4213,8 @@ struct VarArgAArch64Helper : public VarArgHelper { continue; if (!Base) continue; - IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); + IRB.CreateAlignedStore(MSV.getShadow(A), Base, + kShadowTLSAlignment.value()); } Constant *OverflowSize = ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset); @@ -4227,7 +4238,7 @@ struct VarArgAArch64Helper : public VarArgHelper { VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), @@ -4239,7 +4250,7 @@ struct VarArgAArch64Helper : public VarArgHelper { VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), @@ -4280,7 +4291,7 @@ struct VarArgAArch64Helper : public VarArgHelper { IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); } Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize); @@ -4331,14 +4342,15 @@ struct VarArgAArch64Helper : public VarArgHelper { Value *GrRegSaveAreaShadowPtr = MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), - /*Alignment*/ 8, /*isStore*/ true) + Align(8), /*isStore*/ true) .first; Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy, GrRegSaveAreaShadowPtrOff); Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff); - IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize); + IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8), + GrCopySize); // Again, but for FP/SIMD values. Value *VrRegSaveAreaShadowPtrOff = @@ -4346,7 +4358,7 @@ struct VarArgAArch64Helper : public VarArgHelper { Value *VrRegSaveAreaShadowPtr = MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), - /*Alignment*/ 8, /*isStore*/ true) + Align(8), /*isStore*/ true) .first; Value *VrSrcPtr = IRB.CreateInBoundsGEP( @@ -4356,20 +4368,21 @@ struct VarArgAArch64Helper : public VarArgHelper { VrRegSaveAreaShadowPtrOff); Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff); - IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize); + IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8), + VrCopySize); // And finally for remaining arguments. Value *StackSaveAreaShadowPtr = MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(), - /*Alignment*/ 16, /*isStore*/ true) + Align(16), /*isStore*/ true) .first; Value *StackSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset)); - IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16, - VAArgOverflowSize); + IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr, + Align(16), VAArgOverflowSize); } } }; @@ -4461,7 +4474,8 @@ struct VarArgPowerPC64Helper : public VarArgHelper { Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset - VAArgBase, ArgSize); if (Base) - IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); + IRB.CreateAlignedStore(MSV.getShadow(A), Base, + kShadowTLSAlignment.value()); } VAArgOffset += ArgSize; VAArgOffset = alignTo(VAArgOffset, 8); @@ -4494,7 +4508,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper { VAStartInstrumentationList.push_back(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), @@ -4505,7 +4519,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper { IRBuilder<> IRB(&I); Value *VAListTag = I.getArgOperand(0); Value *ShadowPtr, *OriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); // Unpoison the whole __va_list_tag. @@ -4526,7 +4540,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper { // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize); + IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); } // Instrument va_start. @@ -4542,7 +4556,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper { Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; - unsigned Alignment = 8; + const Align Alignment = Align(8); std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); @@ -4595,7 +4609,10 @@ bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) { // Clear out readonly/readnone attributes. AttrBuilder B; B.addAttribute(Attribute::ReadOnly) - .addAttribute(Attribute::ReadNone); + .addAttribute(Attribute::ReadNone) + .addAttribute(Attribute::WriteOnly) + .addAttribute(Attribute::ArgMemOnly) + .addAttribute(Attribute::Speculatable); F.removeAttributes(AttributeList::FunctionIndex, B); return Visitor.runOnFunction(); diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index ca1bb62389e9..cc96bdd1d516 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -47,6 +47,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "CFGMST.h" #include "ValueProfileCollector.h" #include "llvm/ADT/APInt.h" @@ -92,6 +93,7 @@ #include "llvm/IR/ProfileSummary.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" @@ -106,7 +108,6 @@ #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/MisExpect.h" #include <algorithm> diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp index 9f81bb16d0a7..d0afe2959b39 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp @@ -35,6 +35,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/PassRegistry.h" #include "llvm/PassSupport.h" diff --git a/llvm/lib/Transforms/Instrumentation/PoisonChecking.cpp b/llvm/lib/Transforms/Instrumentation/PoisonChecking.cpp index 81d92e724c7d..71ecfd9a2642 100644 --- a/llvm/lib/Transforms/Instrumentation/PoisonChecking.cpp +++ b/llvm/lib/Transforms/Instrumentation/PoisonChecking.cpp @@ -65,10 +65,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/IRBuilder.h" #include "llvm/IR/PatternMatch.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" using namespace llvm; diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index f8fa9cad03b8..e6dc684c2e77 100644 --- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index ac274a155a80..9b7edad3444b 100644 --- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -26,7 +26,6 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" @@ -37,6 +36,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -45,6 +45,7 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/EscapeEnumerator.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; |