summaryrefslogtreecommitdiff
path: root/lib/Transforms/Instrumentation
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2018-07-28 10:51:19 +0000
committerDimitry Andric <dim@FreeBSD.org>2018-07-28 10:51:19 +0000
commiteb11fae6d08f479c0799db45860a98af528fa6e7 (patch)
tree44d492a50c8c1a7eb8e2d17ea3360ec4d066f042 /lib/Transforms/Instrumentation
parentb8a2042aa938069e862750553db0e4d82d25822c (diff)
vendor/llvm/llvm-trunk-r338150
Notes
Diffstat (limited to 'lib/Transforms/Instrumentation')
-rw-r--r--lib/Transforms/Instrumentation/AddressSanitizer.cpp139
-rw-r--r--lib/Transforms/Instrumentation/BoundsChecking.cpp44
-rw-r--r--lib/Transforms/Instrumentation/CFGMST.h16
-rw-r--r--lib/Transforms/Instrumentation/CGProfile.cpp100
-rw-r--r--lib/Transforms/Instrumentation/CMakeLists.txt1
-rw-r--r--lib/Transforms/Instrumentation/DataFlowSanitizer.cpp137
-rw-r--r--lib/Transforms/Instrumentation/EfficiencySanitizer.cpp4
-rw-r--r--lib/Transforms/Instrumentation/GCOVProfiling.cpp301
-rw-r--r--lib/Transforms/Instrumentation/HWAddressSanitizer.cpp509
-rw-r--r--lib/Transforms/Instrumentation/IndirectCallPromotion.cpp30
-rw-r--r--lib/Transforms/Instrumentation/InstrProfiling.cpp46
-rw-r--r--lib/Transforms/Instrumentation/MemorySanitizer.cpp851
-rw-r--r--lib/Transforms/Instrumentation/PGOInstrumentation.cpp83
-rw-r--r--lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp74
-rw-r--r--lib/Transforms/Instrumentation/SanitizerCoverage.cpp23
-rw-r--r--lib/Transforms/Instrumentation/ThreadSanitizer.cpp8
16 files changed, 1745 insertions, 621 deletions
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 8e39f24d819c..b3f659194558 100644
--- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -16,7 +16,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
@@ -25,6 +25,7 @@
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/IR/Argument.h"
@@ -71,7 +72,6 @@
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
@@ -107,10 +107,18 @@ static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
+static const uint64_t kNetBSD_ShadowOffset32 = 1ULL << 30;
static const uint64_t kNetBSD_ShadowOffset64 = 1ULL << 46;
static const uint64_t kPS4CPU_ShadowOffset64 = 1ULL << 40;
static const uint64_t kWindowsShadowOffset32 = 3ULL << 28;
+static const uint64_t kMyriadShadowScale = 5;
+static const uint64_t kMyriadMemoryOffset32 = 0x80000000ULL;
+static const uint64_t kMyriadMemorySize32 = 0x20000000ULL;
+static const uint64_t kMyriadTagShift = 29;
+static const uint64_t kMyriadDDRTag = 4;
+static const uint64_t kMyriadCacheBitMask32 = 0x40000000ULL;
+
// The shadow memory space is dynamically allocated.
static const uint64_t kWindowsShadowOffset64 = kDynamicShadowSentinel;
@@ -145,7 +153,7 @@ static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return";
static const int kMaxAsanStackMallocSizeClass = 10;
static const char *const kAsanStackMallocNameTemplate = "__asan_stack_malloc_";
static const char *const kAsanStackFreeNameTemplate = "__asan_stack_free_";
-static const char *const kAsanGenPrefix = "__asan_gen_";
+static const char *const kAsanGenPrefix = "___asan_gen_";
static const char *const kODRGenPrefix = "__odr_asan_gen_";
static const char *const kSanCovGenPrefix = "__sancov_gen_";
static const char *const kAsanSetShadowPrefix = "__asan_set_shadow_";
@@ -485,18 +493,17 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
bool IsSystemZ = TargetTriple.getArch() == Triple::systemz;
bool IsX86 = TargetTriple.getArch() == Triple::x86;
bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
- bool IsMIPS32 = TargetTriple.getArch() == Triple::mips ||
- TargetTriple.getArch() == Triple::mipsel;
- bool IsMIPS64 = TargetTriple.getArch() == Triple::mips64 ||
- TargetTriple.getArch() == Triple::mips64el;
+ bool IsMIPS32 = TargetTriple.isMIPS32();
+ bool IsMIPS64 = TargetTriple.isMIPS64();
bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb();
bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64;
bool IsWindows = TargetTriple.isOSWindows();
bool IsFuchsia = TargetTriple.isOSFuchsia();
+ bool IsMyriad = TargetTriple.getVendor() == llvm::Triple::Myriad;
ShadowMapping Mapping;
- Mapping.Scale = kDefaultShadowScale;
+ Mapping.Scale = IsMyriad ? kMyriadShadowScale : kDefaultShadowScale;
if (ClMappingScale.getNumOccurrences() > 0) {
Mapping.Scale = ClMappingScale;
}
@@ -508,11 +515,18 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
Mapping.Offset = kMIPS32_ShadowOffset32;
else if (IsFreeBSD)
Mapping.Offset = kFreeBSD_ShadowOffset32;
+ else if (IsNetBSD)
+ Mapping.Offset = kNetBSD_ShadowOffset32;
else if (IsIOS)
// If we're targeting iOS and x86, the binary is built for iOS simulator.
Mapping.Offset = IsX86 ? kIOSSimShadowOffset32 : kIOSShadowOffset32;
else if (IsWindows)
Mapping.Offset = kWindowsShadowOffset32;
+ else if (IsMyriad) {
+ uint64_t ShadowOffset = (kMyriadMemoryOffset32 + kMyriadMemorySize32 -
+ (kMyriadMemorySize32 >> Mapping.Scale));
+ Mapping.Offset = ShadowOffset - (kMyriadMemoryOffset32 >> Mapping.Scale);
+ }
else
Mapping.Offset = kDefaultShadowOffset32;
} else { // LongSize == 64
@@ -589,9 +603,10 @@ struct AddressSanitizer : public FunctionPass {
explicit AddressSanitizer(bool CompileKernel = false, bool Recover = false,
bool UseAfterScope = false)
- : FunctionPass(ID), CompileKernel(CompileKernel || ClEnableKasan),
- Recover(Recover || ClRecover),
- UseAfterScope(UseAfterScope || ClUseAfterScope) {
+ : FunctionPass(ID), UseAfterScope(UseAfterScope || ClUseAfterScope) {
+ this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
+ this->CompileKernel = ClEnableKasan.getNumOccurrences() > 0 ?
+ ClEnableKasan : CompileKernel;
initializeAddressSanitizerPass(*PassRegistry::getPassRegistry());
}
@@ -717,8 +732,7 @@ public:
explicit AddressSanitizerModule(bool CompileKernel = false,
bool Recover = false,
bool UseGlobalsGC = true)
- : ModulePass(ID), CompileKernel(CompileKernel || ClEnableKasan),
- Recover(Recover || ClRecover),
+ : ModulePass(ID),
UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC),
// Not a typo: ClWithComdat is almost completely pointless without
// ClUseGlobalsGC (because then it only works on modules without
@@ -727,7 +741,12 @@ public:
// argument is designed as workaround. Therefore, disable both
// ClWithComdat and ClUseGlobalsGC unless the frontend says it's ok to
// do globals-gc.
- UseCtorComdat(UseGlobalsGC && ClWithComdat) {}
+ UseCtorComdat(UseGlobalsGC && ClWithComdat) {
+ this->Recover = ClRecover.getNumOccurrences() > 0 ?
+ ClRecover : Recover;
+ this->CompileKernel = ClEnableKasan.getNumOccurrences() > 0 ?
+ ClEnableKasan : CompileKernel;
+ }
bool runOnModule(Module &M) override;
StringRef getPassName() const override { return "AddressSanitizerModule"; }
@@ -869,7 +888,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
processStaticAllocas();
if (ClDebugStack) {
- DEBUG(dbgs() << F);
+ LLVM_DEBUG(dbgs() << F);
}
return true;
}
@@ -888,13 +907,13 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
void createDynamicAllocasInitStorage();
// ----------------------- Visitors.
- /// \brief Collect all Ret instructions.
+ /// Collect all Ret instructions.
void visitReturnInst(ReturnInst &RI) { RetVec.push_back(&RI); }
- /// \brief Collect all Resume instructions.
+ /// Collect all Resume instructions.
void visitResumeInst(ResumeInst &RI) { RetVec.push_back(&RI); }
- /// \brief Collect all CatchReturnInst instructions.
+ /// Collect all CatchReturnInst instructions.
void visitCleanupReturnInst(CleanupReturnInst &CRI) { RetVec.push_back(&CRI); }
void unpoisonDynamicAllocasBeforeInst(Instruction *InstBefore,
@@ -942,7 +961,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
// requested memory, but also left, partial and right redzones.
void handleDynamicAllocaCall(AllocaInst *AI);
- /// \brief Collect Alloca instructions we want (and can) handle.
+ /// Collect Alloca instructions we want (and can) handle.
void visitAllocaInst(AllocaInst &AI) {
if (!ASan.isInterestingAlloca(AI)) {
if (AI.isStaticAlloca()) {
@@ -963,7 +982,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
AllocaVec.push_back(&AI);
}
- /// \brief Collect lifetime intrinsic calls to check for use-after-scope
+ /// Collect lifetime intrinsic calls to check for use-after-scope
/// errors.
void visitIntrinsicInst(IntrinsicInst &II) {
Intrinsic::ID ID = II.getIntrinsicID();
@@ -1081,7 +1100,7 @@ static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
return Res;
}
-// \brief Create a constant for Str so that we can pass it to the run-time lib.
+// Create a constant for Str so that we can pass it to the run-time lib.
static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
bool AllowMerging) {
Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
@@ -1095,7 +1114,7 @@ static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
return GV;
}
-/// \brief Create a global describing a source location.
+/// Create a global describing a source location.
static GlobalVariable *createPrivateGlobalForSourceLoc(Module &M,
LocationMetadata MD) {
Constant *LocData[] = {
@@ -1111,7 +1130,7 @@ static GlobalVariable *createPrivateGlobalForSourceLoc(Module &M,
return GV;
}
-/// \brief Check if \p G has been created by a trusted compiler pass.
+/// Check if \p G has been created by a trusted compiler pass.
static bool GlobalWasGeneratedByCompiler(GlobalVariable *G) {
// Do not instrument asan globals.
if (G->getName().startswith(kAsanGenPrefix) ||
@@ -1487,6 +1506,8 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
uint32_t TypeSize, bool IsWrite,
Value *SizeArgument, bool UseCalls,
uint32_t Exp) {
+ bool IsMyriad = TargetTriple.getVendor() == llvm::Triple::Myriad;
+
IRBuilder<> IRB(InsertBefore);
Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
@@ -1501,6 +1522,23 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
return;
}
+ if (IsMyriad) {
+ // Strip the cache bit and do range check.
+ // AddrLong &= ~kMyriadCacheBitMask32
+ AddrLong = IRB.CreateAnd(AddrLong, ~kMyriadCacheBitMask32);
+ // Tag = AddrLong >> kMyriadTagShift
+ Value *Tag = IRB.CreateLShr(AddrLong, kMyriadTagShift);
+ // Tag == kMyriadDDRTag
+ Value *TagCheck =
+ IRB.CreateICmpEQ(Tag, ConstantInt::get(IntptrTy, kMyriadDDRTag));
+
+ TerminatorInst *TagCheckTerm = SplitBlockAndInsertIfThen(
+ TagCheck, InsertBefore, false, MDBuilder(*C).createBranchWeights(1, 100000));
+ assert(cast<BranchInst>(TagCheckTerm)->isUnconditional());
+ IRB.SetInsertPoint(TagCheckTerm);
+ InsertBefore = TagCheckTerm;
+ }
+
Type *ShadowTy =
IntegerType::get(*C, std::max(8U, TypeSize >> Mapping.Scale));
Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
@@ -1609,7 +1647,7 @@ void AddressSanitizerModule::createInitializerPoisonCalls(
bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
Type *Ty = G->getValueType();
- DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
+ LLVM_DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
if (GlobalsMD.get(G).IsBlacklisted) return false;
if (!Ty->isSized()) return false;
@@ -1646,12 +1684,17 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
return false;
}
- // Callbacks put into the CRT initializer/terminator sections
- // should not be instrumented.
+ // On COFF, if the section name contains '$', it is highly likely that the
+ // user is using section sorting to create an array of globals similar to
+ // the way initialization callbacks are registered in .init_array and
+ // .CRT$XCU. The ATL also registers things in .ATL$__[azm]. Adding redzones
+ // to such globals is counterproductive, because the intent is that they
+ // will form an array, and out-of-bounds accesses are expected.
// See https://github.com/google/sanitizers/issues/305
// and http://msdn.microsoft.com/en-US/en-en/library/bb918180(v=vs.120).aspx
- if (Section.startswith(".CRT")) {
- DEBUG(dbgs() << "Ignoring a global initializer callback: " << *G << "\n");
+ if (TargetTriple.isOSBinFormatCOFF() && Section.contains('$')) {
+ LLVM_DEBUG(dbgs() << "Ignoring global in sorted section (contains '$'): "
+ << *G << "\n");
return false;
}
@@ -1668,7 +1711,7 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
// them.
if (ParsedSegment == "__OBJC" ||
(ParsedSegment == "__DATA" && ParsedSection.startswith("__objc_"))) {
- DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
+ LLVM_DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
return false;
}
// See https://github.com/google/sanitizers/issues/32
@@ -1680,13 +1723,13 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
// Therefore there's no point in placing redzones into __DATA,__cfstring.
// Moreover, it causes the linker to crash on OS X 10.7
if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") {
- DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
+ LLVM_DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
return false;
}
// The linker merges the contents of cstring_literals and removes the
// trailing zeroes.
if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) {
- DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
+ LLVM_DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
return false;
}
}
@@ -2153,11 +2196,21 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool
if (ClInitializers && MD.IsDynInit) HasDynamicallyInitializedGlobals = true;
- DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
+ LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
Initializers[i] = Initializer;
}
+ // Add instrumented globals to llvm.compiler.used list to avoid LTO from
+ // ConstantMerge'ing them.
+ SmallVector<GlobalValue *, 16> GlobalsToAddToUsedList;
+ for (size_t i = 0; i < n; i++) {
+ GlobalVariable *G = NewGlobals[i];
+ if (G->getName().empty()) continue;
+ GlobalsToAddToUsedList.push_back(G);
+ }
+ appendToCompilerUsed(M, ArrayRef<GlobalValue *>(GlobalsToAddToUsedList));
+
std::string ELFUniqueModuleId =
(UseGlobalsGC && TargetTriple.isOSBinFormatELF()) ? getUniqueModuleId(&M)
: "";
@@ -2177,7 +2230,7 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool
if (HasDynamicallyInitializedGlobals)
createInitializerPoisonCalls(M, ModuleName);
- DEBUG(dbgs() << M);
+ LLVM_DEBUG(dbgs() << M);
return true;
}
@@ -2247,7 +2300,6 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
const std::string TypeStr = AccessIsWrite ? "store" : "load";
const std::string ExpStr = Exp ? "exp_" : "";
- const std::string SuffixStr = CompileKernel ? "N" : "_n";
const std::string EndingStr = Recover ? "_noabort" : "";
SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
@@ -2259,8 +2311,7 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
}
AsanErrorCallbackSized[AccessIsWrite][Exp] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
- kAsanReportErrorTemplate + ExpStr + TypeStr + SuffixStr +
- EndingStr,
+ kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
FunctionType::get(IRB.getVoidTy(), Args2, false)));
AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] =
@@ -2420,7 +2471,7 @@ bool AddressSanitizer::runOnFunction(Function &F) {
// Leave if the function doesn't need instrumentation.
if (!F.hasFnAttribute(Attribute::SanitizeAddress)) return FunctionModified;
- DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
+ LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
initializeCallbacks(*F.getParent());
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
@@ -2435,7 +2486,7 @@ bool AddressSanitizer::runOnFunction(Function &F) {
// We want to instrument every address only once per basic block (unless there
// are calls between uses).
- SmallSet<Value *, 16> TempsToInstrument;
+ SmallPtrSet<Value *, 16> TempsToInstrument;
SmallVector<Instruction *, 16> ToInstrument;
SmallVector<Instruction *, 8> NoReturnCalls;
SmallVector<BasicBlock *, 16> AllBlocks;
@@ -2494,7 +2545,6 @@ bool AddressSanitizer::runOnFunction(Function &F) {
}
bool UseCalls =
- CompileKernel ||
(ClInstrumentationWithCallsThreshold >= 0 &&
ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold);
const DataLayout &DL = F.getParent()->getDataLayout();
@@ -2534,8 +2584,8 @@ bool AddressSanitizer::runOnFunction(Function &F) {
if (NumInstrumented > 0 || ChangedStack || !NoReturnCalls.empty())
FunctionModified = true;
- DEBUG(dbgs() << "ASAN done instrumenting: " << FunctionModified << " "
- << F << "\n");
+ LLVM_DEBUG(dbgs() << "ASAN done instrumenting: " << FunctionModified << " "
+ << F << "\n");
return FunctionModified;
}
@@ -2710,7 +2760,7 @@ void FunctionStackPoisoner::copyArgsPassedByValToAllocas() {
Arg.replaceAllUsesWith(AI);
uint64_t AllocSize = DL.getTypeAllocSize(Ty);
- IRB.CreateMemCpy(AI, &Arg, AllocSize, Align);
+ IRB.CreateMemCpy(AI, Align, &Arg, Align, AllocSize);
}
}
}
@@ -2851,7 +2901,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
}
auto DescriptionString = ComputeASanStackFrameDescription(SVD);
- DEBUG(dbgs() << DescriptionString << " --- " << L.FrameSize << "\n");
+ LLVM_DEBUG(dbgs() << DescriptionString << " --- " << L.FrameSize << "\n");
uint64_t LocalStackSize = L.FrameSize;
bool DoStackMalloc = ClUseAfterReturn && !ASan.CompileKernel &&
LocalStackSize <= kMaxStackMallocSize;
@@ -3086,7 +3136,8 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) {
} else if (GetElementPtrInst *EP = dyn_cast<GetElementPtrInst>(V)) {
Res = findAllocaForValue(EP->getPointerOperand());
} else {
- DEBUG(dbgs() << "Alloca search canceled on unknown instruction: " << *V << "\n");
+ LLVM_DEBUG(dbgs() << "Alloca search canceled on unknown instruction: " << *V
+ << "\n");
}
if (Res) AllocaForValue[V] = Res;
return Res;
diff --git a/lib/Transforms/Instrumentation/BoundsChecking.cpp b/lib/Transforms/Instrumentation/BoundsChecking.cpp
index be9a22a8681b..e13db08e263c 100644
--- a/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -11,6 +11,7 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetFolder.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/BasicBlock.h"
@@ -59,11 +60,11 @@ template <typename GetTrapBBT>
static bool instrumentMemAccess(Value *Ptr, Value *InstVal,
const DataLayout &DL, TargetLibraryInfo &TLI,
ObjectSizeOffsetEvaluator &ObjSizeEval,
- BuilderTy &IRB,
- GetTrapBBT GetTrapBB) {
+ BuilderTy &IRB, GetTrapBBT GetTrapBB,
+ ScalarEvolution &SE) {
uint64_t NeededSize = DL.getTypeStoreSize(InstVal->getType());
- DEBUG(dbgs() << "Instrument " << *Ptr << " for " << Twine(NeededSize)
- << " bytes\n");
+ LLVM_DEBUG(dbgs() << "Instrument " << *Ptr << " for " << Twine(NeededSize)
+ << " bytes\n");
SizeOffsetEvalType SizeOffset = ObjSizeEval.compute(Ptr);
@@ -79,6 +80,10 @@ static bool instrumentMemAccess(Value *Ptr, Value *InstVal,
Type *IntTy = DL.getIntPtrType(Ptr->getType());
Value *NeededSizeVal = ConstantInt::get(IntTy, NeededSize);
+ auto SizeRange = SE.getUnsignedRange(SE.getSCEV(Size));
+ auto OffsetRange = SE.getUnsignedRange(SE.getSCEV(Offset));
+ auto NeededSizeRange = SE.getUnsignedRange(SE.getSCEV(NeededSizeVal));
+
// three checks are required to ensure safety:
// . Offset >= 0 (since the offset is given from the base ptr)
// . Size >= Offset (unsigned)
@@ -87,10 +92,17 @@ static bool instrumentMemAccess(Value *Ptr, Value *InstVal,
// optimization: if Size >= 0 (signed), skip 1st check
// FIXME: add NSW/NUW here? -- we dont care if the subtraction overflows
Value *ObjSize = IRB.CreateSub(Size, Offset);
- Value *Cmp2 = IRB.CreateICmpULT(Size, Offset);
- Value *Cmp3 = IRB.CreateICmpULT(ObjSize, NeededSizeVal);
+ Value *Cmp2 = SizeRange.getUnsignedMin().uge(OffsetRange.getUnsignedMax())
+ ? ConstantInt::getFalse(Ptr->getContext())
+ : IRB.CreateICmpULT(Size, Offset);
+ Value *Cmp3 = SizeRange.sub(OffsetRange)
+ .getUnsignedMin()
+ .uge(NeededSizeRange.getUnsignedMax())
+ ? ConstantInt::getFalse(Ptr->getContext())
+ : IRB.CreateICmpULT(ObjSize, NeededSizeVal);
Value *Or = IRB.CreateOr(Cmp2, Cmp3);
- if (!SizeCI || SizeCI->getValue().slt(0)) {
+ if ((!SizeCI || SizeCI->getValue().slt(0)) &&
+ !SizeRange.getSignedMin().isNonNegative()) {
Value *Cmp1 = IRB.CreateICmpSLT(Offset, ConstantInt::get(IntTy, 0));
Or = IRB.CreateOr(Cmp1, Or);
}
@@ -123,7 +135,8 @@ static bool instrumentMemAccess(Value *Ptr, Value *InstVal,
return true;
}
-static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI) {
+static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI,
+ ScalarEvolution &SE) {
const DataLayout &DL = F.getParent()->getDataLayout();
ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(),
/*RoundToAlign=*/true);
@@ -168,19 +181,19 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI) {
BuilderTy IRB(Inst->getParent(), BasicBlock::iterator(Inst), TargetFolder(DL));
if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
MadeChange |= instrumentMemAccess(LI->getPointerOperand(), LI, DL, TLI,
- ObjSizeEval, IRB, GetTrapBB);
+ ObjSizeEval, IRB, GetTrapBB, SE);
} else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
MadeChange |=
instrumentMemAccess(SI->getPointerOperand(), SI->getValueOperand(),
- DL, TLI, ObjSizeEval, IRB, GetTrapBB);
+ DL, TLI, ObjSizeEval, IRB, GetTrapBB, SE);
} else if (AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Inst)) {
MadeChange |=
instrumentMemAccess(AI->getPointerOperand(), AI->getCompareOperand(),
- DL, TLI, ObjSizeEval, IRB, GetTrapBB);
+ DL, TLI, ObjSizeEval, IRB, GetTrapBB, SE);
} else if (AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Inst)) {
MadeChange |=
instrumentMemAccess(AI->getPointerOperand(), AI->getValOperand(), DL,
- TLI, ObjSizeEval, IRB, GetTrapBB);
+ TLI, ObjSizeEval, IRB, GetTrapBB, SE);
} else {
llvm_unreachable("unknown Instruction type");
}
@@ -190,8 +203,9 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI) {
PreservedAnalyses BoundsCheckingPass::run(Function &F, FunctionAnalysisManager &AM) {
auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
+ auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
- if (!addBoundsChecking(F, TLI))
+ if (!addBoundsChecking(F, TLI, SE))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
@@ -207,11 +221,13 @@ struct BoundsCheckingLegacyPass : public FunctionPass {
bool runOnFunction(Function &F) override {
auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
- return addBoundsChecking(F, TLI);
+ auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+ return addBoundsChecking(F, TLI, SE);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<TargetLibraryInfoWrapperPass>();
+ AU.addRequired<ScalarEvolutionWrapperPass>();
}
};
} // namespace
diff --git a/lib/Transforms/Instrumentation/CFGMST.h b/lib/Transforms/Instrumentation/CFGMST.h
index 075e5672cff8..cc9b149d0b6a 100644
--- a/lib/Transforms/Instrumentation/CFGMST.h
+++ b/lib/Transforms/Instrumentation/CFGMST.h
@@ -31,7 +31,7 @@
namespace llvm {
-/// \brief An union-find based Minimum Spanning Tree for CFG
+/// An union-find based Minimum Spanning Tree for CFG
///
/// Implements a Union-find algorithm to compute Minimum Spanning Tree
/// for a given CFG.
@@ -97,7 +97,7 @@ public:
// Edges with large weight will be put into MST first so they are less likely
// to be instrumented.
void buildEdges() {
- DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n");
+ LLVM_DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n");
const BasicBlock *Entry = &(F.getEntryBlock());
uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
@@ -107,8 +107,8 @@ public:
// Add a fake edge to the entry.
EntryIncoming = &addEdge(nullptr, Entry, EntryWeight);
- DEBUG(dbgs() << " Edge: from fake node to " << Entry->getName()
- << " w = " << EntryWeight << "\n");
+ LLVM_DEBUG(dbgs() << " Edge: from fake node to " << Entry->getName()
+ << " w = " << EntryWeight << "\n");
// Special handling for single BB functions.
if (succ_empty(Entry)) {
@@ -138,8 +138,8 @@ public:
Weight = BPI->getEdgeProbability(&*BB, TargetBB).scale(scaleFactor);
auto *E = &addEdge(&*BB, TargetBB, Weight);
E->IsCritical = Critical;
- DEBUG(dbgs() << " Edge: from " << BB->getName() << " to "
- << TargetBB->getName() << " w=" << Weight << "\n");
+ LLVM_DEBUG(dbgs() << " Edge: from " << BB->getName() << " to "
+ << TargetBB->getName() << " w=" << Weight << "\n");
// Keep track of entry/exit edges:
if (&*BB == Entry) {
@@ -164,8 +164,8 @@ public:
MaxExitOutWeight = BBWeight;
ExitOutgoing = ExitO;
}
- DEBUG(dbgs() << " Edge: from " << BB->getName() << " to fake exit"
- << " w = " << BBWeight << "\n");
+ LLVM_DEBUG(dbgs() << " Edge: from " << BB->getName() << " to fake exit"
+ << " w = " << BBWeight << "\n");
}
}
diff --git a/lib/Transforms/Instrumentation/CGProfile.cpp b/lib/Transforms/Instrumentation/CGProfile.cpp
new file mode 100644
index 000000000000..9606b3da2475
--- /dev/null
+++ b/lib/Transforms/Instrumentation/CGProfile.cpp
@@ -0,0 +1,100 @@
+//===-- CGProfile.cpp -----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/CGProfile.h"
+
+#include "llvm/ADT/MapVector.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Transforms/Instrumentation.h"
+
+#include <array>
+
+using namespace llvm;
+
+PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) {
+ MapVector<std::pair<Function *, Function *>, uint64_t> Counts;
+ FunctionAnalysisManager &FAM =
+ MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ InstrProfSymtab Symtab;
+ auto UpdateCounts = [&](TargetTransformInfo &TTI, Function *F,
+ Function *CalledF, uint64_t NewCount) {
+ if (!CalledF || !TTI.isLoweredToCall(CalledF))
+ return;
+ uint64_t &Count = Counts[std::make_pair(F, CalledF)];
+ Count = SaturatingAdd(Count, NewCount);
+ };
+ // Ignore error here. Indirect calls are ignored if this fails.
+ (void)(bool)Symtab.create(M);
+ for (auto &F : M) {
+ if (F.isDeclaration())
+ continue;
+ auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
+ if (BFI.getEntryFreq() == 0)
+ continue;
+ TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
+ for (auto &BB : F) {
+ Optional<uint64_t> BBCount = BFI.getBlockProfileCount(&BB);
+ if (!BBCount)
+ continue;
+ for (auto &I : BB) {
+ CallSite CS(&I);
+ if (!CS)
+ continue;
+ if (CS.isIndirectCall()) {
+ InstrProfValueData ValueData[8];
+ uint32_t ActualNumValueData;
+ uint64_t TotalC;
+ if (!getValueProfDataFromInst(*CS.getInstruction(),
+ IPVK_IndirectCallTarget, 8, ValueData,
+ ActualNumValueData, TotalC))
+ continue;
+ for (const auto &VD :
+ ArrayRef<InstrProfValueData>(ValueData, ActualNumValueData)) {
+ UpdateCounts(TTI, &F, Symtab.getFunction(VD.Value), VD.Count);
+ }
+ continue;
+ }
+ UpdateCounts(TTI, &F, CS.getCalledFunction(), *BBCount);
+ }
+ }
+ }
+
+ addModuleFlags(M, Counts);
+
+ return PreservedAnalyses::all();
+}
+
+void CGProfilePass::addModuleFlags(
+ Module &M,
+ MapVector<std::pair<Function *, Function *>, uint64_t> &Counts) const {
+ if (Counts.empty())
+ return;
+
+ LLVMContext &Context = M.getContext();
+ MDBuilder MDB(Context);
+ std::vector<Metadata *> Nodes;
+
+ for (auto E : Counts) {
+ SmallVector<Metadata *, 3> Vals;
+ Vals.push_back(ValueAsMetadata::get(E.first.first));
+ Vals.push_back(ValueAsMetadata::get(E.first.second));
+ Vals.push_back(MDB.createConstant(
+ ConstantInt::get(Type::getInt64Ty(Context), E.second)));
+ Nodes.push_back(MDNode::get(Context, Vals));
+ }
+
+ M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes));
+}
diff --git a/lib/Transforms/Instrumentation/CMakeLists.txt b/lib/Transforms/Instrumentation/CMakeLists.txt
index 66fdcb3ccc49..5d0084823190 100644
--- a/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -1,6 +1,7 @@
add_llvm_library(LLVMInstrumentation
AddressSanitizer.cpp
BoundsChecking.cpp
+ CGProfile.cpp
DataFlowSanitizer.cpp
GCOVProfiling.cpp
MemorySanitizer.cpp
diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 09bcbb282653..bb0e4379d1a8 100644
--- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -56,6 +56,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
@@ -90,7 +91,6 @@
#include "llvm/Support/SpecialCaseList.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -211,6 +211,72 @@ class DFSanABIList {
}
};
+/// TransformedFunction is used to express the result of transforming one
+/// function type into another. This struct is immutable. It holds metadata
+/// useful for updating calls of the old function to the new type.
+struct TransformedFunction {
+ TransformedFunction(FunctionType* OriginalType,
+ FunctionType* TransformedType,
+ std::vector<unsigned> ArgumentIndexMapping)
+ : OriginalType(OriginalType),
+ TransformedType(TransformedType),
+ ArgumentIndexMapping(ArgumentIndexMapping) {}
+
+ // Disallow copies.
+ TransformedFunction(const TransformedFunction&) = delete;
+ TransformedFunction& operator=(const TransformedFunction&) = delete;
+
+ // Allow moves.
+ TransformedFunction(TransformedFunction&&) = default;
+ TransformedFunction& operator=(TransformedFunction&&) = default;
+
+ /// Type of the function before the transformation.
+ FunctionType* const OriginalType;
+
+ /// Type of the function after the transformation.
+ FunctionType* const TransformedType;
+
+ /// Transforming a function may change the position of arguments. This
+ /// member records the mapping from each argument's old position to its new
+ /// position. Argument positions are zero-indexed. If the transformation
+ /// from F to F' made the first argument of F into the third argument of F',
+ /// then ArgumentIndexMapping[0] will equal 2.
+ const std::vector<unsigned> ArgumentIndexMapping;
+};
+
+/// Given function attributes from a call site for the original function,
+/// return function attributes appropriate for a call to the transformed
+/// function.
+AttributeList TransformFunctionAttributes(
+ const TransformedFunction& TransformedFunction,
+ LLVMContext& Ctx, AttributeList CallSiteAttrs) {
+
+ // Construct a vector of AttributeSet for each function argument.
+ std::vector<llvm::AttributeSet> ArgumentAttributes(
+ TransformedFunction.TransformedType->getNumParams());
+
+ // Copy attributes from the parameter of the original function to the
+ // transformed version. 'ArgumentIndexMapping' holds the mapping from
+ // old argument position to new.
+ for (unsigned i=0, ie = TransformedFunction.ArgumentIndexMapping.size();
+ i < ie; ++i) {
+ unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[i];
+ ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttributes(i);
+ }
+
+ // Copy annotations on varargs arguments.
+ for (unsigned i = TransformedFunction.OriginalType->getNumParams(),
+ ie = CallSiteAttrs.getNumAttrSets(); i<ie; ++i) {
+ ArgumentAttributes.push_back(CallSiteAttrs.getParamAttributes(i));
+ }
+
+ return AttributeList::get(
+ Ctx,
+ CallSiteAttrs.getFnAttributes(),
+ CallSiteAttrs.getRetAttributes(),
+ llvm::makeArrayRef(ArgumentAttributes));
+}
+
class DataFlowSanitizer : public ModulePass {
friend struct DFSanFunction;
friend class DFSanVisitor;
@@ -294,7 +360,7 @@ class DataFlowSanitizer : public ModulePass {
bool isInstrumented(const GlobalAlias *GA);
FunctionType *getArgsFunctionType(FunctionType *T);
FunctionType *getTrampolineFunctionType(FunctionType *T);
- FunctionType *getCustomFunctionType(FunctionType *T);
+ TransformedFunction getCustomFunctionType(FunctionType *T);
InstrumentedABI getInstrumentedABI();
WrapperKind getWrapperKind(Function *F);
void addGlobalNamePrefix(GlobalValue *GV);
@@ -437,17 +503,25 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) {
return FunctionType::get(T->getReturnType(), ArgTypes, false);
}
-FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
+TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
SmallVector<Type *, 4> ArgTypes;
- for (FunctionType::param_iterator i = T->param_begin(), e = T->param_end();
- i != e; ++i) {
+
+ // Some parameters of the custom function being constructed are
+ // parameters of T. Record the mapping from parameters of T to
+ // parameters of the custom function, so that parameter attributes
+ // at call sites can be updated.
+ std::vector<unsigned> ArgumentIndexMapping;
+ for (unsigned i = 0, ie = T->getNumParams(); i != ie; ++i) {
+ Type* param_type = T->getParamType(i);
FunctionType *FT;
- if (isa<PointerType>(*i) && (FT = dyn_cast<FunctionType>(cast<PointerType>(
- *i)->getElementType()))) {
+ if (isa<PointerType>(param_type) && (FT = dyn_cast<FunctionType>(
+ cast<PointerType>(param_type)->getElementType()))) {
+ ArgumentIndexMapping.push_back(ArgTypes.size());
ArgTypes.push_back(getTrampolineFunctionType(FT)->getPointerTo());
ArgTypes.push_back(Type::getInt8PtrTy(*Ctx));
} else {
- ArgTypes.push_back(*i);
+ ArgumentIndexMapping.push_back(ArgTypes.size());
+ ArgTypes.push_back(param_type);
}
}
for (unsigned i = 0, e = T->getNumParams(); i != e; ++i)
@@ -457,14 +531,15 @@ FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
Type *RetType = T->getReturnType();
if (!RetType->isVoidTy())
ArgTypes.push_back(ShadowPtrTy);
- return FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg());
+ return TransformedFunction(
+ T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
+ ArgumentIndexMapping);
}
bool DataFlowSanitizer::doInitialization(Module &M) {
Triple TargetTriple(M.getTargetTriple());
bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
- bool IsMIPS64 = TargetTriple.getArch() == Triple::mips64 ||
- TargetTriple.getArch() == Triple::mips64el;
+ bool IsMIPS64 = TargetTriple.isMIPS64();
bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 ||
TargetTriple.getArch() == Triple::aarch64_be;
@@ -783,9 +858,17 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
FunctionType *NewFT = getInstrumentedABI() == IA_Args
? getArgsFunctionType(FT)
: FT;
+
+ // If the function being wrapped has local linkage, then preserve the
+ // function's linkage in the wrapper function.
+ GlobalValue::LinkageTypes wrapperLinkage =
+ F.hasLocalLinkage()
+ ? F.getLinkage()
+ : GlobalValue::LinkOnceODRLinkage;
+
Function *NewF = buildWrapperFunction(
&F, std::string("dfsw$") + std::string(F.getName()),
- GlobalValue::LinkOnceODRLinkage, NewFT);
+ wrapperLinkage, NewFT);
if (getInstrumentedABI() == IA_TLS)
NewF->removeAttributes(AttributeList::FunctionIndex, ReadOnlyNoneAttrs);
@@ -1382,20 +1465,19 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
Value *LenShadow = IRB.CreateMul(
I.getLength(),
ConstantInt::get(I.getLength()->getType(), DFSF.DFS.ShadowWidth / 8));
- Value *AlignShadow;
- if (ClPreserveAlignment) {
- AlignShadow = IRB.CreateMul(I.getAlignmentCst(),
- ConstantInt::get(I.getAlignmentCst()->getType(),
- DFSF.DFS.ShadowWidth / 8));
- } else {
- AlignShadow = ConstantInt::get(I.getAlignmentCst()->getType(),
- DFSF.DFS.ShadowWidth / 8);
- }
Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr);
SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
- IRB.CreateCall(I.getCalledValue(), {DestShadow, SrcShadow, LenShadow,
- AlignShadow, I.getVolatileCst()});
+ auto *MTI = cast<MemTransferInst>(
+ IRB.CreateCall(I.getCalledValue(),
+ {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
+ if (ClPreserveAlignment) {
+ MTI->setDestAlignment(I.getDestAlignment() * (DFSF.DFS.ShadowWidth / 8));
+ MTI->setSourceAlignment(I.getSourceAlignment() * (DFSF.DFS.ShadowWidth / 8));
+ } else {
+ MTI->setDestAlignment(DFSF.DFS.ShadowWidth / 8);
+ MTI->setSourceAlignment(DFSF.DFS.ShadowWidth / 8);
+ }
}
void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
@@ -1460,11 +1542,11 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
// wrapper.
if (CallInst *CI = dyn_cast<CallInst>(CS.getInstruction())) {
FunctionType *FT = F->getFunctionType();
- FunctionType *CustomFT = DFSF.DFS.getCustomFunctionType(FT);
+ TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
std::string CustomFName = "__dfsw_";
CustomFName += F->getName();
- Constant *CustomF =
- DFSF.DFS.Mod->getOrInsertFunction(CustomFName, CustomFT);
+ Constant *CustomF = DFSF.DFS.Mod->getOrInsertFunction(
+ CustomFName, CustomFn.TransformedType);
if (Function *CustomFn = dyn_cast<Function>(CustomF)) {
CustomFn->copyAttributesFrom(F);
@@ -1532,7 +1614,8 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
CustomCI->setCallingConv(CI->getCallingConv());
- CustomCI->setAttributes(CI->getAttributes());
+ CustomCI->setAttributes(TransformFunctionAttributes(CustomFn,
+ CI->getContext(), CI->getAttributes()));
// Update the parameter attributes of the custom call instruction to
// zero extend the shadow parameters. This is required for targets
diff --git a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp b/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
index 6864d295525c..33f220a893df 100644
--- a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
@@ -23,6 +23,7 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -33,7 +34,6 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
@@ -537,7 +537,7 @@ void EfficiencySanitizer::createDestructor(Module &M, Constant *ToolInfoArg) {
bool EfficiencySanitizer::initOnModule(Module &M) {
Triple TargetTriple(M.getTargetTriple());
- if (TargetTriple.getArch() == Triple::mips64 || TargetTriple.getArch() == Triple::mips64el)
+ if (TargetTriple.isMIPS64())
ShadowParams = ShadowParams40;
else
ShadowParams = ShadowParams47;
diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 67ca8172b0d5..acd27c2e226f 100644
--- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -17,11 +17,13 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/UniqueVector.h"
#include "llvm/Analysis/EHPersonalities.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/IRBuilder.h"
@@ -35,8 +37,8 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/GCOVProfiler.h"
#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
#include <memory>
@@ -84,7 +86,7 @@ public:
ReversedVersion[3] = Options.Version[0];
ReversedVersion[4] = '\0';
}
- bool runOnModule(Module &M);
+ bool runOnModule(Module &M, const TargetLibraryInfo &TLI);
private:
// Create the .gcno files for the Module based on DebugInfo.
@@ -130,6 +132,7 @@ private:
SmallVector<uint32_t, 4> FileChecksums;
Module *M;
+ const TargetLibraryInfo *TLI;
LLVMContext *Ctx;
SmallVector<std::unique_ptr<GCOVFunction>, 16> Funcs;
};
@@ -145,7 +148,14 @@ public:
}
StringRef getPassName() const override { return "GCOV Profiler"; }
- bool runOnModule(Module &M) override { return Profiler.runOnModule(M); }
+ bool runOnModule(Module &M) override {
+ auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
+ return Profiler.runOnModule(M, TLI);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetLibraryInfoWrapperPass>();
+ }
private:
GCOVProfiler Profiler;
@@ -153,8 +163,13 @@ private:
}
char GCOVProfilerLegacyPass::ID = 0;
-INITIALIZE_PASS(GCOVProfilerLegacyPass, "insert-gcov-profiling",
- "Insert instrumentation for GCOV profiling", false, false)
+INITIALIZE_PASS_BEGIN(
+ GCOVProfilerLegacyPass, "insert-gcov-profiling",
+ "Insert instrumentation for GCOV profiling", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_END(
+ GCOVProfilerLegacyPass, "insert-gcov-profiling",
+ "Insert instrumentation for GCOV profiling", false, false)
ModulePass *llvm::createGCOVProfilerPass(const GCOVOptions &Options) {
return new GCOVProfilerLegacyPass(Options);
@@ -272,7 +287,7 @@ namespace {
write(Len);
write(Number);
- std::sort(
+ llvm::sort(
SortedLinesByFile.begin(), SortedLinesByFile.end(),
[](StringMapEntry<GCOVLines> *LHS, StringMapEntry<GCOVLines> *RHS) {
return LHS->getKey() < RHS->getKey();
@@ -315,7 +330,7 @@ namespace {
ReturnBlock(1, os) {
this->os = os;
- DEBUG(dbgs() << "Function: " << getFunctionName(SP) << "\n");
+ LLVM_DEBUG(dbgs() << "Function: " << getFunctionName(SP) << "\n");
uint32_t i = 0;
for (auto &BB : *F) {
@@ -383,7 +398,7 @@ namespace {
for (int i = 0, e = Blocks.size() + 1; i != e; ++i) {
write(0); // No flags on our blocks.
}
- DEBUG(dbgs() << Blocks.size() << " blocks.\n");
+ LLVM_DEBUG(dbgs() << Blocks.size() << " blocks.\n");
// Emit edges between blocks.
if (Blocks.empty()) return;
@@ -396,8 +411,8 @@ namespace {
write(Block.OutEdges.size() * 2 + 1);
write(Block.Number);
for (int i = 0, e = Block.OutEdges.size(); i != e; ++i) {
- DEBUG(dbgs() << Block.Number << " -> " << Block.OutEdges[i]->Number
- << "\n");
+ LLVM_DEBUG(dbgs() << Block.Number << " -> "
+ << Block.OutEdges[i]->Number << "\n");
write(Block.OutEdges[i]->Number);
write(0); // no flags
}
@@ -461,8 +476,9 @@ std::string GCOVProfiler::mangleName(const DICompileUnit *CU,
return CurPath.str();
}
-bool GCOVProfiler::runOnModule(Module &M) {
+bool GCOVProfiler::runOnModule(Module &M, const TargetLibraryInfo &TLI) {
this->M = &M;
+ this->TLI = &TLI;
Ctx = &M.getContext();
if (Options.EmitNotes) emitProfileNotes();
@@ -475,7 +491,8 @@ PreservedAnalyses GCOVProfilerPass::run(Module &M,
GCOVProfiler Profiler(GCOVOpts);
- if (!Profiler.runOnModule(M))
+ auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
+ if (!Profiler.runOnModule(M, TLI))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
@@ -503,11 +520,11 @@ static bool functionHasLines(Function &F) {
return false;
}
-static bool isUsingFuncletBasedEH(Function &F) {
+static bool isUsingScopeBasedEH(Function &F) {
if (!F.hasPersonalityFn()) return false;
EHPersonality Personality = classifyEHPersonality(F.getPersonalityFn());
- return isFuncletEHPersonality(Personality);
+ return isScopedEHPersonality(Personality);
}
static bool shouldKeepInEntry(BasicBlock::iterator It) {
@@ -550,8 +567,8 @@ void GCOVProfiler::emitProfileNotes() {
DISubprogram *SP = F.getSubprogram();
if (!SP) continue;
if (!functionHasLines(F)) continue;
- // TODO: Functions using funclet-based EH are currently not supported.
- if (isUsingFuncletBasedEH(F)) continue;
+ // TODO: Functions using scope-based EH are currently not supported.
+ if (isUsingScopeBasedEH(F)) continue;
// gcov expects every function to start with an entry block that has a
// single successor, so split the entry block to make sure of that.
@@ -629,8 +646,8 @@ bool GCOVProfiler::emitProfileArcs() {
DISubprogram *SP = F.getSubprogram();
if (!SP) continue;
if (!functionHasLines(F)) continue;
- // TODO: Functions using funclet-based EH are currently not supported.
- if (isUsingFuncletBasedEH(F)) continue;
+ // TODO: Functions using scope-based EH are currently not supported.
+ if (isUsingScopeBasedEH(F)) continue;
if (!Result) Result = true;
unsigned Edges = 0;
@@ -807,7 +824,12 @@ Constant *GCOVProfiler::getStartFileFunc() {
Type::getInt32Ty(*Ctx), // uint32_t checksum
};
FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
- return M->getOrInsertFunction("llvm_gcda_start_file", FTy);
+ auto *Res = M->getOrInsertFunction("llvm_gcda_start_file", FTy);
+ if (Function *FunRes = dyn_cast<Function>(Res))
+ if (auto AK = TLI->getExtAttrForI32Param(false))
+ FunRes->addParamAttr(2, AK);
+ return Res;
+
}
Constant *GCOVProfiler::getIncrementIndirectCounterFunc() {
@@ -830,7 +852,15 @@ Constant *GCOVProfiler::getEmitFunctionFunc() {
Type::getInt32Ty(*Ctx), // uint32_t cfg_checksum
};
FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
- return M->getOrInsertFunction("llvm_gcda_emit_function", FTy);
+ auto *Res = M->getOrInsertFunction("llvm_gcda_emit_function", FTy);
+ if (Function *FunRes = dyn_cast<Function>(Res))
+ if (auto AK = TLI->getExtAttrForI32Param(false)) {
+ FunRes->addParamAttr(0, AK);
+ FunRes->addParamAttr(2, AK);
+ FunRes->addParamAttr(3, AK);
+ FunRes->addParamAttr(4, AK);
+ }
+ return Res;
}
Constant *GCOVProfiler::getEmitArcsFunc() {
@@ -839,7 +869,11 @@ Constant *GCOVProfiler::getEmitArcsFunc() {
Type::getInt64PtrTy(*Ctx), // uint64_t *counters
};
FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
- return M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy);
+ auto *Res = M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy);
+ if (Function *FunRes = dyn_cast<Function>(Res))
+ if (auto AK = TLI->getExtAttrForI32Param(false))
+ FunRes->addParamAttr(0, AK);
+ return Res;
}
Constant *GCOVProfiler::getSummaryInfoFunc() {
@@ -886,46 +920,205 @@ Function *GCOVProfiler::insertCounterWriteout(
Constant *SummaryInfo = getSummaryInfoFunc();
Constant *EndFile = getEndFileFunc();
- NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
- if (CU_Nodes) {
- for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
- auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
+ NamedMDNode *CUNodes = M->getNamedMetadata("llvm.dbg.cu");
+ if (!CUNodes) {
+ Builder.CreateRetVoid();
+ return WriteoutF;
+ }
- // Skip module skeleton (and module) CUs.
- if (CU->getDWOId())
- continue;
+ // Collect the relevant data into a large constant data structure that we can
+ // walk to write out everything.
+ StructType *StartFileCallArgsTy = StructType::create(
+ {Builder.getInt8PtrTy(), Builder.getInt8PtrTy(), Builder.getInt32Ty()});
+ StructType *EmitFunctionCallArgsTy = StructType::create(
+ {Builder.getInt32Ty(), Builder.getInt8PtrTy(), Builder.getInt32Ty(),
+ Builder.getInt8Ty(), Builder.getInt32Ty()});
+ StructType *EmitArcsCallArgsTy = StructType::create(
+ {Builder.getInt32Ty(), Builder.getInt64Ty()->getPointerTo()});
+ StructType *FileInfoTy =
+ StructType::create({StartFileCallArgsTy, Builder.getInt32Ty(),
+ EmitFunctionCallArgsTy->getPointerTo(),
+ EmitArcsCallArgsTy->getPointerTo()});
- std::string FilenameGcda = mangleName(CU, GCovFileType::GCDA);
- uint32_t CfgChecksum = FileChecksums.empty() ? 0 : FileChecksums[i];
- Builder.CreateCall(StartFile,
- {Builder.CreateGlobalStringPtr(FilenameGcda),
- Builder.CreateGlobalStringPtr(ReversedVersion),
- Builder.getInt32(CfgChecksum)});
- for (unsigned j = 0, e = CountersBySP.size(); j != e; ++j) {
- auto *SP = cast_or_null<DISubprogram>(CountersBySP[j].second);
- uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[j]->getFuncChecksum();
- Builder.CreateCall(
- EmitFunction,
- {Builder.getInt32(j),
- Options.FunctionNamesInData
- ? Builder.CreateGlobalStringPtr(getFunctionName(SP))
- : Constant::getNullValue(Builder.getInt8PtrTy()),
- Builder.getInt32(FuncChecksum),
- Builder.getInt8(Options.UseCfgChecksum),
- Builder.getInt32(CfgChecksum)});
+ Constant *Zero32 = Builder.getInt32(0);
+ // Build an explicit array of two zeros for use in ConstantExpr GEP building.
+ Constant *TwoZero32s[] = {Zero32, Zero32};
- GlobalVariable *GV = CountersBySP[j].first;
- unsigned Arcs =
- cast<ArrayType>(GV->getValueType())->getNumElements();
- Builder.CreateCall(EmitArcs, {Builder.getInt32(Arcs),
- Builder.CreateConstGEP2_64(GV, 0, 0)});
- }
- Builder.CreateCall(SummaryInfo, {});
- Builder.CreateCall(EndFile, {});
+ SmallVector<Constant *, 8> FileInfos;
+ for (int i : llvm::seq<int>(0, CUNodes->getNumOperands())) {
+ auto *CU = cast<DICompileUnit>(CUNodes->getOperand(i));
+
+ // Skip module skeleton (and module) CUs.
+ if (CU->getDWOId())
+ continue;
+
+ std::string FilenameGcda = mangleName(CU, GCovFileType::GCDA);
+ uint32_t CfgChecksum = FileChecksums.empty() ? 0 : FileChecksums[i];
+ auto *StartFileCallArgs = ConstantStruct::get(
+ StartFileCallArgsTy, {Builder.CreateGlobalStringPtr(FilenameGcda),
+ Builder.CreateGlobalStringPtr(ReversedVersion),
+ Builder.getInt32(CfgChecksum)});
+
+ SmallVector<Constant *, 8> EmitFunctionCallArgsArray;
+ SmallVector<Constant *, 8> EmitArcsCallArgsArray;
+ for (int j : llvm::seq<int>(0, CountersBySP.size())) {
+ auto *SP = cast_or_null<DISubprogram>(CountersBySP[j].second);
+ uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[j]->getFuncChecksum();
+ EmitFunctionCallArgsArray.push_back(ConstantStruct::get(
+ EmitFunctionCallArgsTy,
+ {Builder.getInt32(j),
+ Options.FunctionNamesInData
+ ? Builder.CreateGlobalStringPtr(getFunctionName(SP))
+ : Constant::getNullValue(Builder.getInt8PtrTy()),
+ Builder.getInt32(FuncChecksum),
+ Builder.getInt8(Options.UseCfgChecksum),
+ Builder.getInt32(CfgChecksum)}));
+
+ GlobalVariable *GV = CountersBySP[j].first;
+ unsigned Arcs = cast<ArrayType>(GV->getValueType())->getNumElements();
+ EmitArcsCallArgsArray.push_back(ConstantStruct::get(
+ EmitArcsCallArgsTy,
+ {Builder.getInt32(Arcs), ConstantExpr::getInBoundsGetElementPtr(
+ GV->getValueType(), GV, TwoZero32s)}));
}
+ // Create global arrays for the two emit calls.
+ int CountersSize = CountersBySP.size();
+ assert(CountersSize == (int)EmitFunctionCallArgsArray.size() &&
+ "Mismatched array size!");
+ assert(CountersSize == (int)EmitArcsCallArgsArray.size() &&
+ "Mismatched array size!");
+ auto *EmitFunctionCallArgsArrayTy =
+ ArrayType::get(EmitFunctionCallArgsTy, CountersSize);
+ auto *EmitFunctionCallArgsArrayGV = new GlobalVariable(
+ *M, EmitFunctionCallArgsArrayTy, /*isConstant*/ true,
+ GlobalValue::InternalLinkage,
+ ConstantArray::get(EmitFunctionCallArgsArrayTy,
+ EmitFunctionCallArgsArray),
+ Twine("__llvm_internal_gcov_emit_function_args.") + Twine(i));
+ auto *EmitArcsCallArgsArrayTy =
+ ArrayType::get(EmitArcsCallArgsTy, CountersSize);
+ EmitFunctionCallArgsArrayGV->setUnnamedAddr(
+ GlobalValue::UnnamedAddr::Global);
+ auto *EmitArcsCallArgsArrayGV = new GlobalVariable(
+ *M, EmitArcsCallArgsArrayTy, /*isConstant*/ true,
+ GlobalValue::InternalLinkage,
+ ConstantArray::get(EmitArcsCallArgsArrayTy, EmitArcsCallArgsArray),
+ Twine("__llvm_internal_gcov_emit_arcs_args.") + Twine(i));
+ EmitArcsCallArgsArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+
+ FileInfos.push_back(ConstantStruct::get(
+ FileInfoTy,
+ {StartFileCallArgs, Builder.getInt32(CountersSize),
+ ConstantExpr::getInBoundsGetElementPtr(EmitFunctionCallArgsArrayTy,
+ EmitFunctionCallArgsArrayGV,
+ TwoZero32s),
+ ConstantExpr::getInBoundsGetElementPtr(
+ EmitArcsCallArgsArrayTy, EmitArcsCallArgsArrayGV, TwoZero32s)}));
+ }
+
+ // If we didn't find anything to actually emit, bail on out.
+ if (FileInfos.empty()) {
+ Builder.CreateRetVoid();
+ return WriteoutF;
}
+ // To simplify code, we cap the number of file infos we write out to fit
+ // easily in a 32-bit signed integer. This gives consistent behavior between
+ // 32-bit and 64-bit systems without requiring (potentially very slow) 64-bit
+ // operations on 32-bit systems. It also seems unreasonable to try to handle
+ // more than 2 billion files.
+ if ((int64_t)FileInfos.size() > (int64_t)INT_MAX)
+ FileInfos.resize(INT_MAX);
+
+ // Create a global for the entire data structure so we can walk it more
+ // easily.
+ auto *FileInfoArrayTy = ArrayType::get(FileInfoTy, FileInfos.size());
+ auto *FileInfoArrayGV = new GlobalVariable(
+ *M, FileInfoArrayTy, /*isConstant*/ true, GlobalValue::InternalLinkage,
+ ConstantArray::get(FileInfoArrayTy, FileInfos),
+ "__llvm_internal_gcov_emit_file_info");
+ FileInfoArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+
+ // Create the CFG for walking this data structure.
+ auto *FileLoopHeader =
+ BasicBlock::Create(*Ctx, "file.loop.header", WriteoutF);
+ auto *CounterLoopHeader =
+ BasicBlock::Create(*Ctx, "counter.loop.header", WriteoutF);
+ auto *FileLoopLatch = BasicBlock::Create(*Ctx, "file.loop.latch", WriteoutF);
+ auto *ExitBB = BasicBlock::Create(*Ctx, "exit", WriteoutF);
+
+ // We always have at least one file, so just branch to the header.
+ Builder.CreateBr(FileLoopHeader);
+
+ // The index into the files structure is our loop induction variable.
+ Builder.SetInsertPoint(FileLoopHeader);
+ PHINode *IV =
+ Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
+ IV->addIncoming(Builder.getInt32(0), BB);
+ auto *FileInfoPtr =
+ Builder.CreateInBoundsGEP(FileInfoArrayGV, {Builder.getInt32(0), IV});
+ auto *StartFileCallArgsPtr = Builder.CreateStructGEP(FileInfoPtr, 0);
+ auto *StartFileCall = Builder.CreateCall(
+ StartFile,
+ {Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 0)),
+ Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 1)),
+ Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 2))});
+ if (auto AK = TLI->getExtAttrForI32Param(false))
+ StartFileCall->addParamAttr(2, AK);
+ auto *NumCounters =
+ Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 1));
+ auto *EmitFunctionCallArgsArray =
+ Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 2));
+ auto *EmitArcsCallArgsArray =
+ Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 3));
+ auto *EnterCounterLoopCond =
+ Builder.CreateICmpSLT(Builder.getInt32(0), NumCounters);
+ Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
+
+ Builder.SetInsertPoint(CounterLoopHeader);
+ auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
+ JV->addIncoming(Builder.getInt32(0), FileLoopHeader);
+ auto *EmitFunctionCallArgsPtr =
+ Builder.CreateInBoundsGEP(EmitFunctionCallArgsArray, {JV});
+ auto *EmitFunctionCall = Builder.CreateCall(
+ EmitFunction,
+ {Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 0)),
+ Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 1)),
+ Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 2)),
+ Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 3)),
+ Builder.CreateLoad(
+ Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 4))});
+ if (auto AK = TLI->getExtAttrForI32Param(false)) {
+ EmitFunctionCall->addParamAttr(0, AK);
+ EmitFunctionCall->addParamAttr(2, AK);
+ EmitFunctionCall->addParamAttr(3, AK);
+ EmitFunctionCall->addParamAttr(4, AK);
+ }
+ auto *EmitArcsCallArgsPtr =
+ Builder.CreateInBoundsGEP(EmitArcsCallArgsArray, {JV});
+ auto *EmitArcsCall = Builder.CreateCall(
+ EmitArcs,
+ {Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 0)),
+ Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 1))});
+ if (auto AK = TLI->getExtAttrForI32Param(false))
+ EmitArcsCall->addParamAttr(0, AK);
+ auto *NextJV = Builder.CreateAdd(JV, Builder.getInt32(1));
+ auto *CounterLoopCond = Builder.CreateICmpSLT(NextJV, NumCounters);
+ Builder.CreateCondBr(CounterLoopCond, CounterLoopHeader, FileLoopLatch);
+ JV->addIncoming(NextJV, CounterLoopHeader);
+
+ Builder.SetInsertPoint(FileLoopLatch);
+ Builder.CreateCall(SummaryInfo, {});
+ Builder.CreateCall(EndFile, {});
+ auto *NextIV = Builder.CreateAdd(IV, Builder.getInt32(1));
+ auto *FileLoopCond =
+ Builder.CreateICmpSLT(NextIV, Builder.getInt32(FileInfos.size()));
+ Builder.CreateCondBr(FileLoopCond, FileLoopHeader, ExitBB);
+ IV->addIncoming(NextIV, FileLoopLatch);
+
+ Builder.SetInsertPoint(ExitBB);
Builder.CreateRetVoid();
+
return WriteoutF;
}
diff --git a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 8e2833d22032..d62598bb5d4f 100644
--- a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -22,10 +22,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/MDBuilder.h"
-#include "llvm/Support/raw_ostream.h"
#include "llvm/IR/Function.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/InstVisitor.h"
@@ -34,6 +31,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
@@ -41,8 +39,11 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
+#include "llvm/Transforms/Utils/PromoteMemToReg.h"
using namespace llvm;
@@ -51,10 +52,15 @@ using namespace llvm;
static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
static const char *const kHwasanInitName = "__hwasan_init";
+static const char *const kHwasanShadowMemoryDynamicAddress =
+ "__hwasan_shadow_memory_dynamic_address";
+
// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
static const size_t kNumberOfAccessSizes = 5;
-static const size_t kShadowScale = 4;
+static const size_t kDefaultShadowScale = 4;
+static const uint64_t kDynamicShadowSentinel =
+ std::numeric_limits<uint64_t>::max();
static const unsigned kPointerTagShift = 56;
static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
@@ -85,17 +91,57 @@ static cl::opt<bool> ClRecover(
cl::desc("Enable recovery mode (continue-after-error)."),
cl::Hidden, cl::init(false));
+static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
+ cl::desc("instrument stack (allocas)"),
+ cl::Hidden, cl::init(true));
+
+static cl::opt<bool> ClUARRetagToZero(
+ "hwasan-uar-retag-to-zero",
+ cl::desc("Clear alloca tags before returning from the function to allow "
+ "non-instrumented and instrumented function calls mix. When set "
+ "to false, allocas are retagged before returning from the "
+ "function to detect use after return."),
+ cl::Hidden, cl::init(true));
+
+static cl::opt<bool> ClGenerateTagsWithCalls(
+ "hwasan-generate-tags-with-calls",
+ cl::desc("generate new tags with runtime library calls"), cl::Hidden,
+ cl::init(false));
+
+static cl::opt<int> ClMatchAllTag(
+ "hwasan-match-all-tag",
+ cl::desc("don't report bad accesses via pointers with this tag"),
+ cl::Hidden, cl::init(-1));
+
+static cl::opt<bool> ClEnableKhwasan(
+ "hwasan-kernel",
+ cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
+ cl::Hidden, cl::init(false));
+
+// These flags allow to change the shadow mapping and control how shadow memory
+// is accessed. The shadow mapping looks like:
+// Shadow = (Mem >> scale) + offset
+
+static cl::opt<unsigned long long> ClMappingOffset(
+ "hwasan-mapping-offset",
+ cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden,
+ cl::init(0));
+
namespace {
-/// \brief An instrumentation pass implementing detection of addressability bugs
+/// An instrumentation pass implementing detection of addressability bugs
/// using tagged pointers.
class HWAddressSanitizer : public FunctionPass {
public:
// Pass identification, replacement for typeid.
static char ID;
- HWAddressSanitizer(bool Recover = false)
- : FunctionPass(ID), Recover(Recover || ClRecover) {}
+ explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false)
+ : FunctionPass(ID) {
+ this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
+ this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
+ ClEnableKhwasan : CompileKernel;
+ }
StringRef getPassName() const override { return "HWAddressSanitizer"; }
@@ -103,6 +149,11 @@ public:
bool doInitialization(Module &M) override;
void initializeCallbacks(Module &M);
+
+ void maybeInsertDynamicShadowAtFunctionEntry(Function &F);
+
+ void untagPointerOperand(Instruction *I, Value *Addr);
+ Value *memToShadow(Value *Shadow, Type *Ty, IRBuilder<> &IRB);
void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
unsigned AccessSizeIndex,
Instruction *InsertBefore);
@@ -111,16 +162,54 @@ public:
uint64_t *TypeSize, unsigned *Alignment,
Value **MaybeMask);
+ bool isInterestingAlloca(const AllocaInst &AI);
+ bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
+ Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
+ Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
+ bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
+ SmallVectorImpl<Instruction *> &RetVec);
+ Value *getNextTagWithCall(IRBuilder<> &IRB);
+ Value *getStackBaseTag(IRBuilder<> &IRB);
+ Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
+ unsigned AllocaNo);
+ Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
+
private:
LLVMContext *C;
+ Triple TargetTriple;
+
+ /// This struct defines the shadow mapping using the rule:
+ /// shadow = (mem >> Scale) + Offset.
+ /// If InGlobal is true, then
+ /// extern char __hwasan_shadow[];
+ /// shadow = (mem >> Scale) + &__hwasan_shadow
+ struct ShadowMapping {
+ int Scale;
+ uint64_t Offset;
+ bool InGlobal;
+
+ void init(Triple &TargetTriple);
+ unsigned getAllocaAlignment() const { return 1U << Scale; }
+ };
+ ShadowMapping Mapping;
+
Type *IntptrTy;
+ Type *Int8Ty;
+ bool CompileKernel;
bool Recover;
Function *HwasanCtorFunction;
Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
Function *HwasanMemoryAccessCallbackSized[2];
+
+ Function *HwasanTagMemoryFunc;
+ Function *HwasanGenerateTagFunc;
+
+ Constant *ShadowGlobal;
+
+ Value *LocalDynamicShadow = nullptr;
};
} // end anonymous namespace
@@ -129,34 +218,44 @@ char HWAddressSanitizer::ID = 0;
INITIALIZE_PASS_BEGIN(
HWAddressSanitizer, "hwasan",
- "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
+ "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
+ false)
INITIALIZE_PASS_END(
HWAddressSanitizer, "hwasan",
- "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
+ "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
+ false)
-FunctionPass *llvm::createHWAddressSanitizerPass(bool Recover) {
- return new HWAddressSanitizer(Recover);
+FunctionPass *llvm::createHWAddressSanitizerPass(bool CompileKernel,
+ bool Recover) {
+ assert(!CompileKernel || Recover);
+ return new HWAddressSanitizer(CompileKernel, Recover);
}
-/// \brief Module-level initialization.
+/// Module-level initialization.
///
/// inserts a call to __hwasan_init to the module's constructor list.
bool HWAddressSanitizer::doInitialization(Module &M) {
- DEBUG(dbgs() << "Init " << M.getName() << "\n");
+ LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
auto &DL = M.getDataLayout();
- Triple TargetTriple(M.getTargetTriple());
+ TargetTriple = Triple(M.getTargetTriple());
+
+ Mapping.init(TargetTriple);
C = &(M.getContext());
IRBuilder<> IRB(*C);
IntptrTy = IRB.getIntPtrTy(DL);
+ Int8Ty = IRB.getInt8Ty();
- std::tie(HwasanCtorFunction, std::ignore) =
- createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName,
- kHwasanInitName,
- /*InitArgTypes=*/{},
- /*InitArgs=*/{});
- appendToGlobalCtors(M, HwasanCtorFunction, 0);
+ HwasanCtorFunction = nullptr;
+ if (!CompileKernel) {
+ std::tie(HwasanCtorFunction, std::ignore) =
+ createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName,
+ kHwasanInitName,
+ /*InitArgTypes=*/{},
+ /*InitArgs=*/{});
+ appendToGlobalCtors(M, HwasanCtorFunction, 0);
+ }
return true;
}
@@ -168,7 +267,7 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
HwasanMemoryAccessCallbackSized[AccessIsWrite] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
- ClMemoryAccessCallbackPrefix + TypeStr + EndingStr,
+ ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)));
for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
@@ -180,16 +279,50 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)));
}
}
+
+ HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+ "__hwasan_tag_memory", IRB.getVoidTy(), IntptrTy, Int8Ty, IntptrTy));
+ HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
+ M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
+
+ if (Mapping.InGlobal)
+ ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
+ ArrayType::get(IRB.getInt8Ty(), 0));
+}
+
+void HWAddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
+ // Generate code only when dynamic addressing is needed.
+ if (Mapping.Offset != kDynamicShadowSentinel)
+ return;
+
+ IRBuilder<> IRB(&F.front().front());
+ if (Mapping.InGlobal) {
+ // An empty inline asm with input reg == output reg.
+ // An opaque pointer-to-int cast, basically.
+ InlineAsm *Asm = InlineAsm::get(
+ FunctionType::get(IntptrTy, {ShadowGlobal->getType()}, false),
+ StringRef(""), StringRef("=r,0"),
+ /*hasSideEffects=*/false);
+ LocalDynamicShadow = IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+ } else {
+ Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
+ kHwasanShadowMemoryDynamicAddress, IntptrTy);
+ LocalDynamicShadow = IRB.CreateLoad(GlobalDynamicAddress);
+ }
}
Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
- bool *IsWrite,
- uint64_t *TypeSize,
- unsigned *Alignment,
- Value **MaybeMask) {
+ bool *IsWrite,
+ uint64_t *TypeSize,
+ unsigned *Alignment,
+ Value **MaybeMask) {
// Skip memory accesses inserted by another instrumentation.
if (I->getMetadata("nosanitize")) return nullptr;
+ // Do not instrument the load fetching the dynamic shadow address.
+ if (LocalDynamicShadow == I)
+ return nullptr;
+
Value *PtrOperand = nullptr;
const DataLayout &DL = I->getModule()->getDataLayout();
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
@@ -219,7 +352,7 @@ Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
}
if (PtrOperand) {
- // Do not instrument acesses from different address spaces; we cannot deal
+ // Do not instrument accesses from different address spaces; we cannot deal
// with them.
Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
if (PtrTy->getPointerAddressSpace() != 0)
@@ -236,41 +369,103 @@ Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
return PtrOperand;
}
+static unsigned getPointerOperandIndex(Instruction *I) {
+ if (LoadInst *LI = dyn_cast<LoadInst>(I))
+ return LI->getPointerOperandIndex();
+ if (StoreInst *SI = dyn_cast<StoreInst>(I))
+ return SI->getPointerOperandIndex();
+ if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
+ return RMW->getPointerOperandIndex();
+ if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
+ return XCHG->getPointerOperandIndex();
+ report_fatal_error("Unexpected instruction");
+ return -1;
+}
+
static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
size_t Res = countTrailingZeros(TypeSize / 8);
assert(Res < kNumberOfAccessSizes);
return Res;
}
+void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
+ if (TargetTriple.isAArch64())
+ return;
+
+ IRBuilder<> IRB(I);
+ Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+ Value *UntaggedPtr =
+ IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
+ I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
+}
+
+Value *HWAddressSanitizer::memToShadow(Value *Mem, Type *Ty, IRBuilder<> &IRB) {
+ // Mem >> Scale
+ Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
+ if (Mapping.Offset == 0)
+ return Shadow;
+ // (Mem >> Scale) + Offset
+ Value *ShadowBase;
+ if (LocalDynamicShadow)
+ ShadowBase = LocalDynamicShadow;
+ else
+ ShadowBase = ConstantInt::get(Ty, Mapping.Offset);
+ return IRB.CreateAdd(Shadow, ShadowBase);
+}
+
void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
unsigned AccessSizeIndex,
Instruction *InsertBefore) {
IRBuilder<> IRB(InsertBefore);
- Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
- Value *AddrLong =
- IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
- ~(0xFFULL << kPointerTagShift)));
- Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
- Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
+ Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
+ IRB.getInt8Ty());
+ Value *AddrLong = untagPointer(IRB, PtrLong);
+ Value *ShadowLong = memToShadow(AddrLong, PtrLong->getType(), IRB);
+ Value *MemTag =
+ IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
+ int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
+ ClMatchAllTag : (CompileKernel ? 0xFF : -1);
+ if (matchAllTag != -1) {
+ Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
+ ConstantInt::get(PtrTag->getType(), matchAllTag));
+ TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
+ }
+
TerminatorInst *CheckTerm =
SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
MDBuilder(*C).createBranchWeights(1, 100000));
IRB.SetInsertPoint(CheckTerm);
- // The signal handler will find the data address in x0.
- InlineAsm *Asm = InlineAsm::get(
- FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
- "hlt #" +
- itostr(0x100 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
- "{x0}",
- /*hasSideEffects=*/true);
+ const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
+ InlineAsm *Asm;
+ switch (TargetTriple.getArch()) {
+ case Triple::x86_64:
+ // The signal handler will find the data address in rdi.
+ Asm = InlineAsm::get(
+ FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
+ "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
+ "{rdi}",
+ /*hasSideEffects=*/true);
+ break;
+ case Triple::aarch64:
+ case Triple::aarch64_be:
+ // The signal handler will find the data address in x0.
+ Asm = InlineAsm::get(
+ FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
+ "brk #" + itostr(0x900 + AccessInfo),
+ "{x0}",
+ /*hasSideEffects=*/true);
+ break;
+ default:
+ report_fatal_error("unsupported architecture");
+ }
IRB.CreateCall(Asm, PtrLong);
}
bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
- DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
+ LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
bool IsWrite = false;
unsigned Alignment = 0;
uint64_t TypeSize = 0;
@@ -288,7 +483,7 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
if (isPowerOf2_64(TypeSize) &&
(TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
- (Alignment >= (1UL << kShadowScale) || Alignment == 0 ||
+ (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
Alignment >= TypeSize / 8)) {
size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
if (ClInstrumentWithCalls) {
@@ -301,10 +496,197 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
{AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8)});
}
+ untagPointerOperand(I, Addr);
return true;
}
+static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
+ uint64_t ArraySize = 1;
+ if (AI.isArrayAllocation()) {
+ const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
+ assert(CI && "non-constant array size");
+ ArraySize = CI->getZExtValue();
+ }
+ Type *Ty = AI.getAllocatedType();
+ uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
+ return SizeInBytes * ArraySize;
+}
+
+bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
+ Value *Tag) {
+ size_t Size = (getAllocaSizeInBytes(*AI) + Mapping.getAllocaAlignment() - 1) &
+ ~(Mapping.getAllocaAlignment() - 1);
+
+ Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
+ if (ClInstrumentWithCalls) {
+ IRB.CreateCall(HwasanTagMemoryFunc,
+ {IRB.CreatePointerCast(AI, IntptrTy), JustTag,
+ ConstantInt::get(IntptrTy, Size)});
+ } else {
+ size_t ShadowSize = Size >> Mapping.Scale;
+ Value *ShadowPtr = IRB.CreateIntToPtr(
+ memToShadow(IRB.CreatePointerCast(AI, IntptrTy), AI->getType(), IRB),
+ IRB.getInt8PtrTy());
+ // If this memset is not inlined, it will be intercepted in the hwasan
+ // runtime library. That's OK, because the interceptor skips the checks if
+ // the address is in the shadow region.
+ // FIXME: the interceptor is not as fast as real memset. Consider lowering
+ // llvm.memset right here into either a sequence of stores, or a call to
+ // hwasan_tag_memory.
+ IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
+ }
+ return true;
+}
+
+static unsigned RetagMask(unsigned AllocaNo) {
+ // A list of 8-bit numbers that have at most one run of non-zero bits.
+ // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
+ // masks.
+ // The list does not include the value 255, which is used for UAR.
+ static unsigned FastMasks[] = {
+ 0, 1, 2, 3, 4, 6, 7, 8, 12, 14, 15, 16, 24,
+ 28, 30, 31, 32, 48, 56, 60, 62, 63, 64, 96, 112, 120,
+ 124, 126, 127, 128, 192, 224, 240, 248, 252, 254};
+ return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
+}
+
+Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
+ return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
+}
+
+Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
+ if (ClGenerateTagsWithCalls)
+ return nullptr;
+ // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
+ // first).
+ Module *M = IRB.GetInsertBlock()->getParent()->getParent();
+ auto GetStackPointerFn =
+ Intrinsic::getDeclaration(M, Intrinsic::frameaddress);
+ Value *StackPointer = IRB.CreateCall(
+ GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
+
+ // Extract some entropy from the stack pointer for the tags.
+ // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
+ // between functions).
+ Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
+ Value *StackTag =
+ IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
+ "hwasan.stack.base.tag");
+ return StackTag;
+}
+
+Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
+ AllocaInst *AI, unsigned AllocaNo) {
+ if (ClGenerateTagsWithCalls)
+ return getNextTagWithCall(IRB);
+ return IRB.CreateXor(StackTag,
+ ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
+}
+
+Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
+ if (ClUARRetagToZero)
+ return ConstantInt::get(IntptrTy, 0);
+ if (ClGenerateTagsWithCalls)
+ return getNextTagWithCall(IRB);
+ return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
+}
+
+// Add a tag to an address.
+Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
+ Value *PtrLong, Value *Tag) {
+ Value *TaggedPtrLong;
+ if (CompileKernel) {
+ // Kernel addresses have 0xFF in the most significant byte.
+ Value *ShiftedTag = IRB.CreateOr(
+ IRB.CreateShl(Tag, kPointerTagShift),
+ ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
+ TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
+ } else {
+ // Userspace can simply do OR (tag << 56);
+ Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
+ TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
+ }
+ return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
+}
+
+// Remove tag from an address.
+Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
+ Value *UntaggedPtrLong;
+ if (CompileKernel) {
+ // Kernel addresses have 0xFF in the most significant byte.
+ UntaggedPtrLong = IRB.CreateOr(PtrLong,
+ ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
+ } else {
+ // Userspace addresses have 0x00.
+ UntaggedPtrLong = IRB.CreateAnd(PtrLong,
+ ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
+ }
+ return UntaggedPtrLong;
+}
+
+bool HWAddressSanitizer::instrumentStack(
+ SmallVectorImpl<AllocaInst *> &Allocas,
+ SmallVectorImpl<Instruction *> &RetVec) {
+ Function *F = Allocas[0]->getParent()->getParent();
+ Instruction *InsertPt = &*F->getEntryBlock().begin();
+ IRBuilder<> IRB(InsertPt);
+
+ Value *StackTag = getStackBaseTag(IRB);
+
+ // Ideally, we want to calculate tagged stack base pointer, and rewrite all
+ // alloca addresses using that. Unfortunately, offsets are not known yet
+ // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
+ // temp, shift-OR it into each alloca address and xor with the retag mask.
+ // This generates one extra instruction per alloca use.
+ for (unsigned N = 0; N < Allocas.size(); ++N) {
+ auto *AI = Allocas[N];
+ IRB.SetInsertPoint(AI->getNextNode());
+
+ // Replace uses of the alloca with tagged address.
+ Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
+ Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
+ Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
+ std::string Name =
+ AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
+ Replacement->setName(Name + ".hwasan");
+
+ for (auto UI = AI->use_begin(), UE = AI->use_end(); UI != UE;) {
+ Use &U = *UI++;
+ if (U.getUser() != AILong)
+ U.set(Replacement);
+ }
+
+ tagAlloca(IRB, AI, Tag);
+
+ for (auto RI : RetVec) {
+ IRB.SetInsertPoint(RI);
+
+ // Re-tag alloca memory with the special UAR tag.
+ Value *Tag = getUARTag(IRB, StackTag);
+ tagAlloca(IRB, AI, Tag);
+ }
+ }
+
+ return true;
+}
+
+bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
+ return (AI.getAllocatedType()->isSized() &&
+ // FIXME: instrument dynamic allocas, too
+ AI.isStaticAlloca() &&
+ // alloca() may be called with 0 size, ignore it.
+ getAllocaSizeInBytes(AI) > 0 &&
+ // We are only interested in allocas not promotable to registers.
+ // Promotable allocas are common under -O0.
+ !isAllocaPromotable(&AI) &&
+ // inalloca allocas are not treated as static, and we don't want
+ // dynamic alloca instrumentation for them as well.
+ !AI.isUsedWithInAlloca() &&
+ // swifterror allocas are register promoted by ISel
+ !AI.isSwiftError());
+}
+
bool HWAddressSanitizer::runOnFunction(Function &F) {
if (&F == HwasanCtorFunction)
return false;
@@ -312,14 +694,35 @@ bool HWAddressSanitizer::runOnFunction(Function &F) {
if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
return false;
- DEBUG(dbgs() << "Function: " << F.getName() << "\n");
+ LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
initializeCallbacks(*F.getParent());
+ assert(!LocalDynamicShadow);
+ maybeInsertDynamicShadowAtFunctionEntry(F);
+
bool Changed = false;
SmallVector<Instruction*, 16> ToInstrument;
+ SmallVector<AllocaInst*, 8> AllocasToInstrument;
+ SmallVector<Instruction*, 8> RetVec;
for (auto &BB : F) {
for (auto &Inst : BB) {
+ if (ClInstrumentStack)
+ if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
+ // Realign all allocas. We don't want small uninteresting allocas to
+ // hide in instrumented alloca's padding.
+ if (AI->getAlignment() < Mapping.getAllocaAlignment())
+ AI->setAlignment(Mapping.getAllocaAlignment());
+ // Instrument some of them.
+ if (isInterestingAlloca(*AI))
+ AllocasToInstrument.push_back(AI);
+ continue;
+ }
+
+ if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
+ isa<CleanupReturnInst>(Inst))
+ RetVec.push_back(&Inst);
+
Value *MaybeMask = nullptr;
bool IsWrite;
unsigned Alignment;
@@ -331,8 +734,30 @@ bool HWAddressSanitizer::runOnFunction(Function &F) {
}
}
+ if (!AllocasToInstrument.empty())
+ Changed |= instrumentStack(AllocasToInstrument, RetVec);
+
for (auto Inst : ToInstrument)
Changed |= instrumentMemAccess(Inst);
+ LocalDynamicShadow = nullptr;
+
return Changed;
}
+
+void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
+ const bool IsAndroid = TargetTriple.isAndroid();
+ const bool IsAndroidWithIfuncSupport =
+ IsAndroid && !TargetTriple.isAndroidVersionLT(21);
+
+ Scale = kDefaultShadowScale;
+
+ if (ClEnableKhwasan || ClInstrumentWithCalls || !IsAndroidWithIfuncSupport)
+ Offset = 0;
+ else
+ Offset = kDynamicShadowSentinel;
+ if (ClMappingOffset.getNumOccurrences() > 0)
+ Offset = ClMappingOffset;
+
+ InGlobal = IsAndroidWithIfuncSupport;
+}
diff --git a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 49b8a67a6c14..27fb0e4393af 100644
--- a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -45,7 +45,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/Transforms/PGOInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include <cassert>
@@ -223,12 +223,12 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
uint64_t TotalCount, uint32_t NumCandidates) {
std::vector<PromotionCandidate> Ret;
- DEBUG(dbgs() << " \nWork on callsite #" << NumOfPGOICallsites << *Inst
- << " Num_targets: " << ValueDataRef.size()
- << " Num_candidates: " << NumCandidates << "\n");
+ LLVM_DEBUG(dbgs() << " \nWork on callsite #" << NumOfPGOICallsites << *Inst
+ << " Num_targets: " << ValueDataRef.size()
+ << " Num_candidates: " << NumCandidates << "\n");
NumOfPGOICallsites++;
if (ICPCSSkip != 0 && NumOfPGOICallsites <= ICPCSSkip) {
- DEBUG(dbgs() << " Skip: User options.\n");
+ LLVM_DEBUG(dbgs() << " Skip: User options.\n");
return Ret;
}
@@ -236,11 +236,11 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
uint64_t Count = ValueDataRef[I].Count;
assert(Count <= TotalCount);
uint64_t Target = ValueDataRef[I].Value;
- DEBUG(dbgs() << " Candidate " << I << " Count=" << Count
- << " Target_func: " << Target << "\n");
+ LLVM_DEBUG(dbgs() << " Candidate " << I << " Count=" << Count
+ << " Target_func: " << Target << "\n");
if (ICPInvokeOnly && dyn_cast<CallInst>(Inst)) {
- DEBUG(dbgs() << " Not promote: User options.\n");
+ LLVM_DEBUG(dbgs() << " Not promote: User options.\n");
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst)
<< " Not promote: User options";
@@ -248,7 +248,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
break;
}
if (ICPCallOnly && dyn_cast<InvokeInst>(Inst)) {
- DEBUG(dbgs() << " Not promote: User option.\n");
+ LLVM_DEBUG(dbgs() << " Not promote: User option.\n");
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst)
<< " Not promote: User options";
@@ -256,7 +256,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
break;
}
if (ICPCutOff != 0 && NumOfPGOICallPromotion >= ICPCutOff) {
- DEBUG(dbgs() << " Not promote: Cutoff reached.\n");
+ LLVM_DEBUG(dbgs() << " Not promote: Cutoff reached.\n");
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "CutOffReached", Inst)
<< " Not promote: Cutoff reached";
@@ -266,7 +266,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
Function *TargetFunction = Symtab->getFunction(Target);
if (TargetFunction == nullptr) {
- DEBUG(dbgs() << " Not promote: Cannot find the target\n");
+ LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", Inst)
<< "Cannot promote indirect call: target not found";
@@ -387,7 +387,7 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI,
InstrProfSymtab Symtab;
if (Error E = Symtab.create(M, InLTO)) {
std::string SymtabFailure = toString(std::move(E));
- DEBUG(dbgs() << "Failed to create symtab: " << SymtabFailure << "\n");
+ LLVM_DEBUG(dbgs() << "Failed to create symtab: " << SymtabFailure << "\n");
(void)SymtabFailure;
return false;
}
@@ -412,12 +412,12 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI,
ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO, *ORE);
bool FuncChanged = ICallPromotion.processFunction(PSI);
if (ICPDUMPAFTER && FuncChanged) {
- DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
- DEBUG(dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
}
Changed |= FuncChanged;
if (ICPCutOff != 0 && NumOfPGOICallPromotion >= ICPCutOff) {
- DEBUG(dbgs() << " Stop: Cutoff reached.\n");
+ LLVM_DEBUG(dbgs() << " Stop: Cutoff reached.\n");
break;
}
}
diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 9b70f95480e4..22076f04d6ad 100644
--- a/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -13,7 +13,7 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/InstrProfiling.h"
+#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
@@ -271,8 +271,8 @@ public:
break;
}
- DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
- << L.getLoopDepth() << ")\n");
+ LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
+ << L.getLoopDepth() << ")\n");
return Promoted != 0;
}
@@ -430,9 +430,24 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) {
}
}
-bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
- bool MadeChange = false;
+/// Check if the module contains uses of any profiling intrinsics.
+static bool containsProfilingIntrinsics(Module &M) {
+ if (auto *F = M.getFunction(
+ Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
+ if (!F->use_empty())
+ return true;
+ if (auto *F = M.getFunction(
+ Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
+ if (!F->use_empty())
+ return true;
+ if (auto *F = M.getFunction(
+ Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
+ if (!F->use_empty())
+ return true;
+ return false;
+}
+bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
this->M = &M;
this->TLI = &TLI;
NamesVar = nullptr;
@@ -443,6 +458,15 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
MemOPSizeRangeLast);
TT = Triple(M.getTargetTriple());
+ // Emit the runtime hook even if no counters are present.
+ bool MadeChange = emitRuntimeHook();
+
+ // Improve compile time by avoiding linear scans when there is no work.
+ GlobalVariable *CoverageNamesVar =
+ M.getNamedGlobal(getCoverageUnusedNamesVarName());
+ if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
+ return MadeChange;
+
// We did not know how many value sites there would be inside
// the instrumented function. This is counting the number of instrumented
// target value sites to enter it as field in the profile data variable.
@@ -464,8 +488,7 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
for (Function &F : M)
MadeChange |= lowerIntrinsics(&F);
- if (GlobalVariable *CoverageNamesVar =
- M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
+ if (CoverageNamesVar) {
lowerCoverageData(CoverageNamesVar);
MadeChange = true;
}
@@ -476,7 +499,6 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
emitVNodes();
emitNameData();
emitRegistration();
- emitRuntimeHook();
emitUses();
emitInitialization();
return true;
@@ -669,6 +691,7 @@ static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
// Use linker script magic to get data/cnts/name start/end.
if (Triple(M.getTargetTriple()).isOSLinux() ||
Triple(M.getTargetTriple()).isOSFreeBSD() ||
+ Triple(M.getTargetTriple()).isOSFuchsia() ||
Triple(M.getTargetTriple()).isPS4CPU())
return false;
@@ -892,15 +915,15 @@ void InstrProfiling::emitRegistration() {
IRB.CreateRetVoid();
}
-void InstrProfiling::emitRuntimeHook() {
+bool InstrProfiling::emitRuntimeHook() {
// We expect the linker to be invoked with -u<hook_var> flag for linux,
// for which case there is no need to emit the user function.
if (Triple(M->getTargetTriple()).isOSLinux())
- return;
+ return false;
// If the module's provided its own runtime, we don't need to do anything.
if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
- return;
+ return false;
// Declare an external variable that will pull in the runtime initialization.
auto *Int32Ty = Type::getInt32Ty(M->getContext());
@@ -925,6 +948,7 @@ void InstrProfiling::emitRuntimeHook() {
// Mark the user variable as used so that it isn't stripped out.
UsedVars.push_back(User);
+ return true;
}
void InstrProfiling::emitUses() {
diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index b3c39b5b1665..4bcef6972786 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -101,6 +101,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
@@ -138,7 +139,6 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
#include <cassert>
@@ -163,7 +163,7 @@ static const unsigned kRetvalTLSSize = 800;
// Accesses sizes are powers of two: 1, 2, 4, 8.
static const size_t kNumberOfAccessSizes = 4;
-/// \brief Track origins of uninitialized values.
+/// Track origins of uninitialized values.
///
/// Adds a section to MemorySanitizer report that points to the allocation
/// (stack or heap) the uninitialized bits came from originally.
@@ -199,6 +199,18 @@ static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
cl::desc("exact handling of relational integer ICmp"),
cl::Hidden, cl::init(false));
+// When compiling the Linux kernel, we sometimes see false positives related to
+// MSan being unable to understand that inline assembly calls may initialize
+// local variables.
+// This flag makes the compiler conservatively unpoison every memory location
+// passed into an assembly call. Note that this may cause false positives.
+// Because it's impossible to figure out the array sizes, we can only unpoison
+// the first sizeof(type) bytes for each type* pointer.
+static cl::opt<bool> ClHandleAsmConservative(
+ "msan-handle-asm-conservative",
+ cl::desc("conservative handling of inline assembly"), cl::Hidden,
+ cl::init(false));
+
// This flag controls whether we check the shadow of the address
// operand of load or store. Such bugs are very rare, since load from
// a garbage address typically results in SEGV, but still happen
@@ -234,6 +246,24 @@ static cl::opt<bool> ClWithComdat("msan-with-comdat",
cl::desc("Place MSan constructors in comdat sections"),
cl::Hidden, cl::init(false));
+// These options allow to specify custom memory map parameters
+// See MemoryMapParams for details.
+static cl::opt<unsigned long long> ClAndMask("msan-and-mask",
+ cl::desc("Define custom MSan AndMask"),
+ cl::Hidden, cl::init(0));
+
+static cl::opt<unsigned long long> ClXorMask("msan-xor-mask",
+ cl::desc("Define custom MSan XorMask"),
+ cl::Hidden, cl::init(0));
+
+static cl::opt<unsigned long long> ClShadowBase("msan-shadow-base",
+ cl::desc("Define custom MSan ShadowBase"),
+ cl::Hidden, cl::init(0));
+
+static cl::opt<unsigned long long> ClOriginBase("msan-origin-base",
+ cl::desc("Define custom MSan OriginBase"),
+ cl::Hidden, cl::init(0));
+
static const char *const kMsanModuleCtorName = "msan.module_ctor";
static const char *const kMsanInitName = "__msan_init";
@@ -360,7 +390,7 @@ static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
namespace {
-/// \brief An instrumentation pass implementing detection of uninitialized
+/// An instrumentation pass implementing detection of uninitialized
/// reads.
///
/// MemorySanitizer: instrument the code in module to find
@@ -368,7 +398,7 @@ namespace {
class MemorySanitizer : public FunctionPass {
public:
// Pass identification, replacement for typeid.
- static char ID;
+ static char ID;
MemorySanitizer(int TrackOrigins = 0, bool Recover = false)
: FunctionPass(ID),
@@ -392,8 +422,9 @@ private:
friend struct VarArgPowerPC64Helper;
void initializeCallbacks(Module &M);
+ void createUserspaceApi(Module &M);
- /// \brief Track origins (allocation points) of uninitialized values.
+ /// Track origins (allocation points) of uninitialized values.
int TrackOrigins;
bool Recover;
@@ -401,60 +432,67 @@ private:
Type *IntptrTy;
Type *OriginTy;
- /// \brief Thread-local shadow storage for function parameters.
+ /// Thread-local shadow storage for function parameters.
GlobalVariable *ParamTLS;
- /// \brief Thread-local origin storage for function parameters.
+ /// Thread-local origin storage for function parameters.
GlobalVariable *ParamOriginTLS;
- /// \brief Thread-local shadow storage for function return value.
+ /// Thread-local shadow storage for function return value.
GlobalVariable *RetvalTLS;
- /// \brief Thread-local origin storage for function return value.
+ /// Thread-local origin storage for function return value.
GlobalVariable *RetvalOriginTLS;
- /// \brief Thread-local shadow storage for in-register va_arg function
+ /// Thread-local shadow storage for in-register va_arg function
/// parameters (x86_64-specific).
GlobalVariable *VAArgTLS;
- /// \brief Thread-local shadow storage for va_arg overflow area
+ /// Thread-local shadow storage for va_arg overflow area
/// (x86_64-specific).
GlobalVariable *VAArgOverflowSizeTLS;
- /// \brief Thread-local space used to pass origin value to the UMR reporting
+ /// Thread-local space used to pass origin value to the UMR reporting
/// function.
GlobalVariable *OriginTLS;
- /// \brief The run-time callback to print a warning.
- Value *WarningFn = nullptr;
+ /// Are the instrumentation callbacks set up?
+ bool CallbacksInitialized = false;
+
+ /// The run-time callback to print a warning.
+ Value *WarningFn;
// These arrays are indexed by log2(AccessSize).
Value *MaybeWarningFn[kNumberOfAccessSizes];
Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
- /// \brief Run-time helper that generates a new origin value for a stack
+ /// Run-time helper that generates a new origin value for a stack
/// allocation.
Value *MsanSetAllocaOrigin4Fn;
- /// \brief Run-time helper that poisons stack on function entry.
+ /// Run-time helper that poisons stack on function entry.
Value *MsanPoisonStackFn;
- /// \brief Run-time helper that records a store (or any event) of an
+ /// Run-time helper that records a store (or any event) of an
/// uninitialized value and returns an updated origin id encoding this info.
Value *MsanChainOriginFn;
- /// \brief MSan runtime replacements for memmove, memcpy and memset.
+ /// MSan runtime replacements for memmove, memcpy and memset.
Value *MemmoveFn, *MemcpyFn, *MemsetFn;
- /// \brief Memory map parameters used in application-to-shadow calculation.
+ /// Memory map parameters used in application-to-shadow calculation.
const MemoryMapParams *MapParams;
+ /// Custom memory map parameters used when -msan-shadow-base or
+ // -msan-origin-base is provided.
+ MemoryMapParams CustomMapParams;
+
MDNode *ColdCallWeights;
- /// \brief Branch weights for origin store.
+ /// Branch weights for origin store.
MDNode *OriginStoreWeights;
- /// \brief An empty volatile inline asm that prevents callback merge.
+ /// An empty volatile inline asm that prevents callback merge.
InlineAsm *EmptyAsm;
Function *MsanCtorFunction;
@@ -476,7 +514,7 @@ FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) {
return new MemorySanitizer(TrackOrigins, Recover);
}
-/// \brief Create a non-const global initialized with the given string.
+/// Create a non-const global initialized with the given string.
///
/// Creates a writable global for Str so that we can pass it to the
/// run-time lib. Runtime uses first 4 bytes of the string to store the
@@ -488,12 +526,8 @@ static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
GlobalValue::PrivateLinkage, StrConst, "");
}
-/// \brief Insert extern declaration of runtime-provided functions and globals.
-void MemorySanitizer::initializeCallbacks(Module &M) {
- // Only do this once.
- if (WarningFn)
- return;
-
+/// Insert declarations for userspace-specific functions and globals.
+void MemorySanitizer::createUserspaceApi(Module &M) {
IRBuilder<> IRB(*C);
// Create the callback.
// FIXME: this function should have "Cold" calling conv,
@@ -502,6 +536,38 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
: "__msan_warning_noreturn";
WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
+ // Create the global TLS variables.
+ RetvalTLS = new GlobalVariable(
+ M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
+ GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
+ GlobalVariable::InitialExecTLSModel);
+
+ RetvalOriginTLS = new GlobalVariable(
+ M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
+ "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+
+ ParamTLS = new GlobalVariable(
+ M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
+ GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
+ GlobalVariable::InitialExecTLSModel);
+
+ ParamOriginTLS = new GlobalVariable(
+ M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
+ GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
+ nullptr, GlobalVariable::InitialExecTLSModel);
+
+ VAArgTLS = new GlobalVariable(
+ M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
+ GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
+ GlobalVariable::InitialExecTLSModel);
+ VAArgOverflowSizeTLS = new GlobalVariable(
+ M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
+ "__msan_va_arg_overflow_size_tls", nullptr,
+ GlobalVariable::InitialExecTLSModel);
+ OriginTLS = new GlobalVariable(
+ M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
+ "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+
for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
AccessSizeIndex++) {
unsigned AccessSize = 1 << AccessSizeIndex;
@@ -522,6 +588,17 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
MsanPoisonStackFn =
M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
IRB.getInt8PtrTy(), IntptrTy);
+}
+
+/// Insert extern declaration of runtime-provided functions and globals.
+void MemorySanitizer::initializeCallbacks(Module &M) {
+ // Only do this once.
+ if (CallbacksInitialized)
+ return;
+
+ IRBuilder<> IRB(*C);
+ // Initialize callbacks that are common for kernel and userspace
+ // instrumentation.
MsanChainOriginFn = M.getOrInsertFunction(
"__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
MemmoveFn = M.getOrInsertFunction(
@@ -533,98 +610,81 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
MemsetFn = M.getOrInsertFunction(
"__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
IntptrTy);
-
- // Create globals.
- RetvalTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
- GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
- GlobalVariable::InitialExecTLSModel);
- RetvalOriginTLS = new GlobalVariable(
- M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
- "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
-
- ParamTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
- GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
- GlobalVariable::InitialExecTLSModel);
- ParamOriginTLS = new GlobalVariable(
- M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
- GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
- nullptr, GlobalVariable::InitialExecTLSModel);
-
- VAArgTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
- GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
- GlobalVariable::InitialExecTLSModel);
- VAArgOverflowSizeTLS = new GlobalVariable(
- M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
- "__msan_va_arg_overflow_size_tls", nullptr,
- GlobalVariable::InitialExecTLSModel);
- OriginTLS = new GlobalVariable(
- M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
- "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
-
// We insert an empty inline asm after __msan_report* to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
/*hasSideEffects=*/true);
+
+ createUserspaceApi(M);
+ CallbacksInitialized = true;
}
-/// \brief Module-level initialization.
+/// Module-level initialization.
///
/// inserts a call to __msan_init to the module's constructor list.
bool MemorySanitizer::doInitialization(Module &M) {
auto &DL = M.getDataLayout();
- Triple TargetTriple(M.getTargetTriple());
- switch (TargetTriple.getOS()) {
- case Triple::FreeBSD:
- switch (TargetTriple.getArch()) {
- case Triple::x86_64:
- MapParams = FreeBSD_X86_MemoryMapParams.bits64;
- break;
- case Triple::x86:
- MapParams = FreeBSD_X86_MemoryMapParams.bits32;
- break;
- default:
- report_fatal_error("unsupported architecture");
- }
- break;
- case Triple::NetBSD:
- switch (TargetTriple.getArch()) {
- case Triple::x86_64:
- MapParams = NetBSD_X86_MemoryMapParams.bits64;
- break;
- default:
- report_fatal_error("unsupported architecture");
- }
- break;
- case Triple::Linux:
- switch (TargetTriple.getArch()) {
- case Triple::x86_64:
- MapParams = Linux_X86_MemoryMapParams.bits64;
- break;
- case Triple::x86:
- MapParams = Linux_X86_MemoryMapParams.bits32;
- break;
- case Triple::mips64:
- case Triple::mips64el:
- MapParams = Linux_MIPS_MemoryMapParams.bits64;
- break;
- case Triple::ppc64:
- case Triple::ppc64le:
- MapParams = Linux_PowerPC_MemoryMapParams.bits64;
- break;
- case Triple::aarch64:
- case Triple::aarch64_be:
- MapParams = Linux_ARM_MemoryMapParams.bits64;
- break;
- default:
- report_fatal_error("unsupported architecture");
- }
- break;
- default:
- report_fatal_error("unsupported operating system");
+ bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
+ bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
+ // Check the overrides first
+ if (ShadowPassed || OriginPassed) {
+ CustomMapParams.AndMask = ClAndMask;
+ CustomMapParams.XorMask = ClXorMask;
+ CustomMapParams.ShadowBase = ClShadowBase;
+ CustomMapParams.OriginBase = ClOriginBase;
+ MapParams = &CustomMapParams;
+ } else {
+ Triple TargetTriple(M.getTargetTriple());
+ switch (TargetTriple.getOS()) {
+ case Triple::FreeBSD:
+ switch (TargetTriple.getArch()) {
+ case Triple::x86_64:
+ MapParams = FreeBSD_X86_MemoryMapParams.bits64;
+ break;
+ case Triple::x86:
+ MapParams = FreeBSD_X86_MemoryMapParams.bits32;
+ break;
+ default:
+ report_fatal_error("unsupported architecture");
+ }
+ break;
+ case Triple::NetBSD:
+ switch (TargetTriple.getArch()) {
+ case Triple::x86_64:
+ MapParams = NetBSD_X86_MemoryMapParams.bits64;
+ break;
+ default:
+ report_fatal_error("unsupported architecture");
+ }
+ break;
+ case Triple::Linux:
+ switch (TargetTriple.getArch()) {
+ case Triple::x86_64:
+ MapParams = Linux_X86_MemoryMapParams.bits64;
+ break;
+ case Triple::x86:
+ MapParams = Linux_X86_MemoryMapParams.bits32;
+ break;
+ case Triple::mips64:
+ case Triple::mips64el:
+ MapParams = Linux_MIPS_MemoryMapParams.bits64;
+ break;
+ case Triple::ppc64:
+ case Triple::ppc64le:
+ MapParams = Linux_PowerPC_MemoryMapParams.bits64;
+ break;
+ case Triple::aarch64:
+ case Triple::aarch64_be:
+ MapParams = Linux_ARM_MemoryMapParams.bits64;
+ break;
+ default:
+ report_fatal_error("unsupported architecture");
+ }
+ break;
+ default:
+ report_fatal_error("unsupported operating system");
+ }
}
C = &(M.getContext());
@@ -661,7 +721,7 @@ bool MemorySanitizer::doInitialization(Module &M) {
namespace {
-/// \brief A helper class that handles instrumentation of VarArg
+/// A helper class that handles instrumentation of VarArg
/// functions on a particular platform.
///
/// Implementations are expected to insert the instrumentation
@@ -672,16 +732,16 @@ namespace {
struct VarArgHelper {
virtual ~VarArgHelper() = default;
- /// \brief Visit a CallSite.
+ /// Visit a CallSite.
virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
- /// \brief Visit a va_start call.
+ /// Visit a va_start call.
virtual void visitVAStartInst(VAStartInst &I) = 0;
- /// \brief Visit a va_copy call.
+ /// Visit a va_copy call.
virtual void visitVACopyInst(VACopyInst &I) = 0;
- /// \brief Finalize function instrumentation.
+ /// Finalize function instrumentation.
///
/// This method is called after visiting all interesting (see above)
/// instructions in a function.
@@ -715,6 +775,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ValueMap<Value*, Value*> ShadowMap, OriginMap;
std::unique_ptr<VarArgHelper> VAHelper;
const TargetLibraryInfo *TLI;
+ BasicBlock *ActualFnStart;
// The following flags disable parts of MSan instrumentation based on
// blacklist contents and command-line options.
@@ -747,9 +808,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
CheckReturnValue = SanitizeFunction && (F.getName() == "main");
TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
- DEBUG(if (!InsertChecks)
- dbgs() << "MemorySanitizer is not inserting checks into '"
- << F.getName() << "'\n");
+ MS.initializeCallbacks(*F.getParent());
+ ActualFnStart = &F.getEntryBlock();
+
+ LLVM_DEBUG(if (!InsertChecks) dbgs()
+ << "MemorySanitizer is not inserting checks into '"
+ << F.getName() << "'\n");
}
Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
@@ -766,7 +830,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
}
- /// \brief Fill memory range with the given origin value.
+ /// Fill memory range with the given origin value.
void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
unsigned Size, unsigned Alignment) {
const DataLayout &DL = F.getParent()->getDataLayout();
@@ -849,13 +913,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
unsigned Alignment = SI->getAlignment();
unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
std::tie(ShadowPtr, OriginPtr) =
- getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment);
+ getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
- DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
-
- if (ClCheckAccessAddress)
- insertShadowCheck(Addr, NewSI);
+ LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
+ (void)NewSI;
if (SI->isAtomic())
SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
@@ -866,25 +928,31 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
+ /// Helper function to insert a warning at IRB's current insert point.
+ void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
+ if (!Origin)
+ Origin = (Value *)IRB.getInt32(0);
+ if (MS.TrackOrigins) {
+ IRB.CreateStore(Origin, MS.OriginTLS);
+ }
+ IRB.CreateCall(MS.WarningFn, {});
+ IRB.CreateCall(MS.EmptyAsm, {});
+ // FIXME: Insert UnreachableInst if !MS.Recover?
+ // This may invalidate some of the following checks and needs to be done
+ // at the very end.
+ }
+
void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
bool AsCall) {
IRBuilder<> IRB(OrigIns);
- DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
+ LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
- DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
+ LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
if (ConstantShadow) {
if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
- if (MS.TrackOrigins) {
- IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
- MS.OriginTLS);
- }
- IRB.CreateCall(MS.WarningFn, {});
- IRB.CreateCall(MS.EmptyAsm, {});
- // FIXME: Insert UnreachableInst if !MS.Recover?
- // This may invalidate some of the following checks and needs to be done
- // at the very end.
+ insertWarningFn(IRB, Origin);
}
return;
}
@@ -908,13 +976,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
/* Unreachable */ !MS.Recover, MS.ColdCallWeights);
IRB.SetInsertPoint(CheckTerm);
- if (MS.TrackOrigins) {
- IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
- MS.OriginTLS);
- }
- IRB.CreateCall(MS.WarningFn, {});
- IRB.CreateCall(MS.EmptyAsm, {});
- DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
+ insertWarningFn(IRB, Origin);
+ LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
}
}
@@ -925,13 +988,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Value *Origin = ShadowData.Origin;
materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
}
- DEBUG(dbgs() << "DONE:\n" << F);
+ LLVM_DEBUG(dbgs() << "DONE:\n" << F);
}
- /// \brief Add MemorySanitizer instrumentation to a function.
+ /// Add MemorySanitizer instrumentation to a function.
bool runOnFunction() {
- MS.initializeCallbacks(*F.getParent());
-
// In the presence of unreachable blocks, we may see Phi nodes with
// incoming nodes from such blocks. Since InstVisitor skips unreachable
// blocks, such nodes will not have any shadow value associated with them.
@@ -941,7 +1002,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Iterate all BBs in depth-first order and create shadow instructions
// for all instructions (where applicable).
// For PHI nodes we create dummy shadow PHIs which will be finalized later.
- for (BasicBlock *BB : depth_first(&F.getEntryBlock()))
+ for (BasicBlock *BB : depth_first(ActualFnStart))
visit(*BB);
// Finalize PHI nodes.
@@ -961,22 +1022,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
InstrumentationList.size() + StoreList.size() >
(unsigned)ClInstrumentationWithCallThreshold;
- // Delayed instrumentation of StoreInst.
- // This may add new checks to be inserted later.
- materializeStores(InstrumentWithCalls);
-
// Insert shadow value checks.
materializeChecks(InstrumentWithCalls);
+ // Delayed instrumentation of StoreInst.
+ // This may not add new address checks.
+ materializeStores(InstrumentWithCalls);
+
return true;
}
- /// \brief Compute the shadow type that corresponds to a given Value.
+ /// Compute the shadow type that corresponds to a given Value.
Type *getShadowTy(Value *V) {
return getShadowTy(V->getType());
}
- /// \brief Compute the shadow type that corresponds to a given Type.
+ /// Compute the shadow type that corresponds to a given Type.
Type *getShadowTy(Type *OrigTy) {
if (!OrigTy->isSized()) {
return nullptr;
@@ -1000,21 +1061,21 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
Elements.push_back(getShadowTy(ST->getElementType(i)));
StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
- DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
+ LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
return Res;
}
uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
return IntegerType::get(*MS.C, TypeSize);
}
- /// \brief Flatten a vector type.
+ /// Flatten a vector type.
Type *getShadowTyNoVec(Type *ty) {
if (VectorType *vt = dyn_cast<VectorType>(ty))
return IntegerType::get(*MS.C, vt->getBitWidth());
return ty;
}
- /// \brief Convert a shadow value to it's flattened variant.
+ /// Convert a shadow value to it's flattened variant.
Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
Type *Ty = V->getType();
Type *NoVecTy = getShadowTyNoVec(Ty);
@@ -1022,7 +1083,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return IRB.CreateBitCast(V, NoVecTy);
}
- /// \brief Compute the integer shadow offset that corresponds to a given
+ /// Compute the integer shadow offset that corresponds to a given
/// application address.
///
/// Offset = (Addr & ~AndMask) ^ XorMask
@@ -1041,18 +1102,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return OffsetLong;
}
- /// \brief Compute the shadow and origin addresses corresponding to a given
+ /// Compute the shadow and origin addresses corresponding to a given
/// application address.
///
/// Shadow = ShadowBase + Offset
/// Origin = (OriginBase + Offset) & ~3ULL
- std::pair<Value *, Value *> getShadowOriginPtrUserspace(
- Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, unsigned Alignment,
- Instruction **FirstInsn) {
+ std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr,
+ IRBuilder<> &IRB,
+ Type *ShadowTy,
+ unsigned Alignment) {
Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
Value *ShadowLong = ShadowOffset;
uint64_t ShadowBase = MS.MapParams->ShadowBase;
- *FirstInsn = dyn_cast<Instruction>(ShadowLong);
if (ShadowBase != 0) {
ShadowLong =
IRB.CreateAdd(ShadowLong,
@@ -1080,58 +1141,60 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
Type *ShadowTy,
- unsigned Alignment) {
- Instruction *FirstInsn = nullptr;
+ unsigned Alignment,
+ bool isStore) {
std::pair<Value *, Value *> ret =
- getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment, &FirstInsn);
+ getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
return ret;
}
- /// \brief Compute the shadow address for a given function argument.
+ /// Compute the shadow address for a given function argument.
///
/// Shadow = ParamTLS+ArgOffset.
Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
int ArgOffset) {
Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
- Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
+ if (ArgOffset)
+ Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
"_msarg");
}
- /// \brief Compute the origin address for a given function argument.
+ /// Compute the origin address for a given function argument.
Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
int ArgOffset) {
if (!MS.TrackOrigins) return nullptr;
Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
- Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
+ if (ArgOffset)
+ Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
"_msarg_o");
}
- /// \brief Compute the shadow address for a retval.
+ /// Compute the shadow address for a retval.
Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
return IRB.CreatePointerCast(MS.RetvalTLS,
PointerType::get(getShadowTy(A), 0),
"_msret");
}
- /// \brief Compute the origin address for a retval.
+ /// Compute the origin address for a retval.
Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
// We keep a single origin for the entire retval. Might be too optimistic.
return MS.RetvalOriginTLS;
}
- /// \brief Set SV to be the shadow value for V.
+ /// Set SV to be the shadow value for V.
void setShadow(Value *V, Value *SV) {
assert(!ShadowMap.count(V) && "Values may only have one shadow");
ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
}
- /// \brief Set Origin to be the origin value for V.
+ /// Set Origin to be the origin value for V.
void setOrigin(Value *V, Value *Origin) {
if (!MS.TrackOrigins) return;
assert(!OriginMap.count(V) && "Values may only have one origin");
- DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
+ LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
OriginMap[V] = Origin;
}
@@ -1142,7 +1205,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return Constant::getNullValue(ShadowTy);
}
- /// \brief Create a clean shadow value for a given value.
+ /// Create a clean shadow value for a given value.
///
/// Clean shadow (all zeroes) means all bits of the value are defined
/// (initialized).
@@ -1150,7 +1213,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return getCleanShadow(V->getType());
}
- /// \brief Create a dirty shadow of a given shadow type.
+ /// Create a dirty shadow of a given shadow type.
Constant *getPoisonedShadow(Type *ShadowTy) {
assert(ShadowTy);
if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
@@ -1169,7 +1232,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
llvm_unreachable("Unexpected shadow type");
}
- /// \brief Create a dirty shadow for a given value.
+ /// Create a dirty shadow for a given value.
Constant *getPoisonedShadow(Value *V) {
Type *ShadowTy = getShadowTy(V);
if (!ShadowTy)
@@ -1177,12 +1240,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return getPoisonedShadow(ShadowTy);
}
- /// \brief Create a clean (zero) origin.
+ /// Create a clean (zero) origin.
Value *getCleanOrigin() {
return Constant::getNullValue(MS.OriginTy);
}
- /// \brief Get the shadow value for a given Value.
+ /// Get the shadow value for a given Value.
///
/// This function either returns the value set earlier with setShadow,
/// or extracts if from ParamTLS (for function arguments).
@@ -1194,7 +1257,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// For instructions the shadow is already stored in the map.
Value *Shadow = ShadowMap[V];
if (!Shadow) {
- DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
+ LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
(void)I;
assert(Shadow && "No shadow for a value");
}
@@ -1202,7 +1265,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
if (UndefValue *U = dyn_cast<UndefValue>(V)) {
Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
- DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
+ LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
(void)U;
return AllOnes;
}
@@ -1212,12 +1275,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (*ShadowPtr)
return *ShadowPtr;
Function *F = A->getParent();
- IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI());
+ IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
unsigned ArgOffset = 0;
const DataLayout &DL = F->getParent()->getDataLayout();
for (auto &FArg : F->args()) {
if (!FArg.getType()->isSized()) {
- DEBUG(dbgs() << "Arg is not sized\n");
+ LLVM_DEBUG(dbgs() << "Arg is not sized\n");
continue;
}
unsigned Size =
@@ -1237,7 +1300,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ArgAlign = DL.getABITypeAlignment(EltType);
}
Value *CpShadowPtr =
- getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign)
+ getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
+ /*isStore*/ true)
.first;
if (Overflow) {
// ParamTLS overflow.
@@ -1246,9 +1310,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Size, ArgAlign);
} else {
unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
- Value *Cpy =
- EntryIRB.CreateMemCpy(CpShadowPtr, Base, Size, CopyAlign);
- DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
+ Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
+ CopyAlign, Size);
+ LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
(void)Cpy;
}
*ShadowPtr = getCleanShadow(V);
@@ -1261,8 +1325,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
}
}
- DEBUG(dbgs() << " ARG: " << FArg << " ==> " <<
- **ShadowPtr << "\n");
+ LLVM_DEBUG(dbgs()
+ << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n");
if (MS.TrackOrigins && !Overflow) {
Value *OriginPtr =
getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
@@ -1280,12 +1344,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return getCleanShadow(V);
}
- /// \brief Get the shadow for i-th argument of the instruction I.
+ /// Get the shadow for i-th argument of the instruction I.
Value *getShadow(Instruction *I, int i) {
return getShadow(I->getOperand(i));
}
- /// \brief Get the origin for a value.
+ /// Get the origin for a value.
Value *getOrigin(Value *V) {
if (!MS.TrackOrigins) return nullptr;
if (!PropagateShadow) return getCleanOrigin();
@@ -1301,12 +1365,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return Origin;
}
- /// \brief Get the origin for i-th argument of the instruction I.
+ /// Get the origin for i-th argument of the instruction I.
Value *getOrigin(Instruction *I, int i) {
return getOrigin(I->getOperand(i));
}
- /// \brief Remember the place where a shadow check should be inserted.
+ /// Remember the place where a shadow check should be inserted.
///
/// This location will be later instrumented with a check that will print a
/// UMR warning in runtime if the shadow value is not 0.
@@ -1322,7 +1386,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
}
- /// \brief Remember the place where a shadow check should be inserted.
+ /// Remember the place where a shadow check should be inserted.
///
/// This location will be later instrumented with a check that will print a
/// UMR warning in runtime if the value is not fully defined.
@@ -1382,7 +1446,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
InstVisitor<MemorySanitizerVisitor>::visit(I);
}
- /// \brief Instrument LoadInst
+ /// Instrument LoadInst
///
/// Loads the corresponding shadow and (optionally) origin.
/// Optionally, checks that the load address is fully defined.
@@ -1396,7 +1460,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
unsigned Alignment = I.getAlignment();
if (PropagateShadow) {
std::tie(ShadowPtr, OriginPtr) =
- getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment);
+ getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
} else {
setShadow(&I, getCleanShadow(&I));
@@ -1418,12 +1482,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
- /// \brief Instrument StoreInst
+ /// Instrument StoreInst
///
/// Stores the corresponding shadow and (optionally) origin.
/// Optionally, checks that the store address is fully defined.
void visitStoreInst(StoreInst &I) {
StoreList.push_back(&I);
+ if (ClCheckAccessAddress)
+ insertShadowCheck(I.getPointerOperand(), &I);
}
void handleCASOrRMW(Instruction &I) {
@@ -1431,8 +1497,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRBuilder<> IRB(&I);
Value *Addr = I.getOperand(0);
- Value *ShadowPtr =
- getShadowOriginPtr(Addr, IRB, I.getType(), /*Alignment*/ 1).first;
+ Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(),
+ /*Alignment*/ 1, /*isStore*/ true)
+ .first;
if (ClCheckAccessAddress)
insertShadowCheck(Addr, &I);
@@ -1536,7 +1603,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
- /// \brief Propagate shadow for bitwise AND.
+ /// Propagate shadow for bitwise AND.
///
/// This code is exact, i.e. if, for example, a bit in the left argument
/// is defined and 0, then neither the value not definedness of the
@@ -1585,7 +1652,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- /// \brief Default propagation of shadow and/or origin.
+ /// Default propagation of shadow and/or origin.
///
/// This class implements the general case of shadow propagation, used in all
/// cases where we don't know and/or don't care about what the operation
@@ -1611,7 +1678,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
: IRB(IRB), MSV(MSV) {}
- /// \brief Add a pair of shadow and origin values to the mix.
+ /// Add a pair of shadow and origin values to the mix.
Combiner &Add(Value *OpShadow, Value *OpOrigin) {
if (CombineShadow) {
assert(OpShadow);
@@ -1641,14 +1708,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return *this;
}
- /// \brief Add an application value to the mix.
+ /// Add an application value to the mix.
Combiner &Add(Value *V) {
Value *OpShadow = MSV->getShadow(V);
Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
return Add(OpShadow, OpOrigin);
}
- /// \brief Set the current combined values as the given instruction's shadow
+ /// Set the current combined values as the given instruction's shadow
/// and origin.
void Done(Instruction *I) {
if (CombineShadow) {
@@ -1666,7 +1733,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
using ShadowAndOriginCombiner = Combiner<true>;
using OriginCombiner = Combiner<false>;
- /// \brief Propagate origin for arbitrary operation.
+ /// Propagate origin for arbitrary operation.
void setOriginForNaryOp(Instruction &I) {
if (!MS.TrackOrigins) return;
IRBuilder<> IRB(&I);
@@ -1684,7 +1751,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Ty->getPrimitiveSizeInBits();
}
- /// \brief Cast between two shadow types, extending or truncating as
+ /// Cast between two shadow types, extending or truncating as
/// necessary.
Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
bool Signed = false) {
@@ -1706,7 +1773,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// TODO: handle struct types.
}
- /// \brief Cast an application value to the type of its own shadow.
+ /// Cast an application value to the type of its own shadow.
Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
Type *ShadowTy = getShadowTy(V);
if (V->getType() == ShadowTy)
@@ -1717,7 +1784,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return IRB.CreateBitCast(V, ShadowTy);
}
- /// \brief Propagate shadow for arbitrary operation.
+ /// Propagate shadow for arbitrary operation.
void handleShadowOr(Instruction &I) {
IRBuilder<> IRB(&I);
ShadowAndOriginCombiner SC(this, IRB);
@@ -1726,7 +1793,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
SC.Done(&I);
}
- // \brief Handle multiplication by constant.
+ // Handle multiplication by constant.
//
// Handle a special case of multiplication by constant that may have one or
// more zeros in the lower bits. This makes corresponding number of lower bits
@@ -1788,7 +1855,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void visitSub(BinaryOperator &I) { handleShadowOr(I); }
void visitXor(BinaryOperator &I) { handleShadowOr(I); }
- void handleDiv(Instruction &I) {
+ void handleIntegerDiv(Instruction &I) {
IRBuilder<> IRB(&I);
// Strict on the second argument.
insertShadowCheck(I.getOperand(1), &I);
@@ -1796,14 +1863,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(&I, 0));
}
- void visitUDiv(BinaryOperator &I) { handleDiv(I); }
- void visitSDiv(BinaryOperator &I) { handleDiv(I); }
- void visitFDiv(BinaryOperator &I) { handleDiv(I); }
- void visitURem(BinaryOperator &I) { handleDiv(I); }
- void visitSRem(BinaryOperator &I) { handleDiv(I); }
- void visitFRem(BinaryOperator &I) { handleDiv(I); }
+ void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
+ void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
+ void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
+ void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
- /// \brief Instrument == and != comparisons.
+ // Floating point division is side-effect free. We can not require that the
+ // divisor is fully initialized and must propagate shadow. See PR37523.
+ void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
+ void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
+
+ /// Instrument == and != comparisons.
///
/// Sometimes the comparison result is known even if some of the bits of the
/// arguments are not.
@@ -1841,7 +1911,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- /// \brief Build the lowest possible value of V, taking into account V's
+ /// Build the lowest possible value of V, taking into account V's
/// uninitialized bits.
Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
bool isSigned) {
@@ -1858,7 +1928,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
- /// \brief Build the highest possible value of V, taking into account V's
+ /// Build the highest possible value of V, taking into account V's
/// uninitialized bits.
Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
bool isSigned) {
@@ -1875,7 +1945,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
- /// \brief Instrument relational comparisons.
+ /// Instrument relational comparisons.
///
/// This function does exact shadow propagation for all relational
/// comparisons of integers, pointers and vectors of those.
@@ -1908,7 +1978,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- /// \brief Instrument signed relational comparisons.
+ /// Instrument signed relational comparisons.
///
/// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
/// bit of the shadow. Everything else is delegated to handleShadowOr().
@@ -1992,7 +2062,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void visitAShr(BinaryOperator &I) { handleShift(I); }
void visitLShr(BinaryOperator &I) { handleShift(I); }
- /// \brief Instrument llvm.memmove
+ /// Instrument llvm.memmove
///
/// At this point we don't know if llvm.memmove will be inlined or not.
/// If we don't instrument it and it gets inlined,
@@ -2045,7 +2115,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
VAHelper->visitVACopyInst(I);
}
- /// \brief Handle vector store-like intrinsics.
+ /// Handle vector store-like intrinsics.
///
/// Instrument intrinsics that look like a simple SIMD store: writes memory,
/// has 1 pointer argument and 1 vector argument, returns void.
@@ -2057,8 +2127,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// We don't know the pointer alignment (could be unaligned SSE store!).
// Have to assume to worst case.
- std::tie(ShadowPtr, OriginPtr) =
- getShadowOriginPtr(Addr, IRB, Shadow->getType(), /*Alignment*/ 1);
+ std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
+ Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true);
IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
if (ClCheckAccessAddress)
@@ -2069,7 +2139,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return true;
}
- /// \brief Handle vector load-like intrinsics.
+ /// Handle vector load-like intrinsics.
///
/// Instrument intrinsics that look like a simple SIMD load: reads memory,
/// has 1 pointer argument, returns a vector.
@@ -2084,7 +2154,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Have to assume to worst case.
unsigned Alignment = 1;
std::tie(ShadowPtr, OriginPtr) =
- getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment);
+ getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
} else {
setShadow(&I, getCleanShadow(&I));
@@ -2102,7 +2172,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return true;
}
- /// \brief Handle (SIMD arithmetic)-like intrinsics.
+ /// Handle (SIMD arithmetic)-like intrinsics.
///
/// Instrument intrinsics with any number of arguments of the same type,
/// equal to the return type. The type should be simple (no aggregates or
@@ -2132,7 +2202,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return true;
}
- /// \brief Heuristically instrument unknown intrinsics.
+ /// Heuristically instrument unknown intrinsics.
///
/// The main purpose of this code is to do something reasonable with all
/// random intrinsics we might encounter, most importantly - SIMD intrinsics.
@@ -2182,7 +2252,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(Op));
}
- // \brief Instrument vector convert instrinsic.
+ // Instrument vector convert instrinsic.
//
// This function instruments intrinsics like cvtsi2ss:
// %Out = int_xxx_cvtyyy(%ConvertOp)
@@ -2285,7 +2355,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return IRB.CreateSExt(S2, T);
}
- // \brief Instrument vector shift instrinsic.
+ // Instrument vector shift instrinsic.
//
// This function instruments intrinsics like int_x86_avx2_psll_w.
// Intrinsic shifts %In by %ShiftSize bits.
@@ -2310,14 +2380,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // \brief Get an X86_MMX-sized vector type.
+ // Get an X86_MMX-sized vector type.
Type *getMMXVectorTy(unsigned EltSizeInBits) {
const unsigned X86_MMXSizeInBits = 64;
return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
X86_MMXSizeInBits / EltSizeInBits);
}
- // \brief Returns a signed counterpart for an (un)signed-saturate-and-pack
+ // Returns a signed counterpart for an (un)signed-saturate-and-pack
// intrinsic.
Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
switch (id) {
@@ -2348,7 +2418,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
- // \brief Instrument vector pack instrinsic.
+ // Instrument vector pack instrinsic.
//
// This function instruments intrinsics like x86_mmx_packsswb, that
// packs elements of 2 input vectors into half as many bits with saturation.
@@ -2391,7 +2461,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // \brief Instrument sum-of-absolute-differencies intrinsic.
+ // Instrument sum-of-absolute-differencies intrinsic.
void handleVectorSadIntrinsic(IntrinsicInst &I) {
const unsigned SignificantBitsPerResultElement = 16;
bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
@@ -2410,7 +2480,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // \brief Instrument multiply-add intrinsic.
+ // Instrument multiply-add intrinsic.
void handleVectorPmaddIntrinsic(IntrinsicInst &I,
unsigned EltSizeInBits = 0) {
bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
@@ -2425,7 +2495,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // \brief Instrument compare-packed intrinsic.
+ // Instrument compare-packed intrinsic.
// Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
// all-ones shadow.
void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
@@ -2438,7 +2508,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // \brief Instrument compare-scalar intrinsic.
+ // Instrument compare-scalar intrinsic.
// This handles both cmp* intrinsics which return the result in the first
// element of a vector, and comi* which return the result as i32.
void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
@@ -2453,7 +2523,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRBuilder<> IRB(&I);
Value* Addr = I.getArgOperand(0);
Type *Ty = IRB.getInt32Ty();
- Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1).first;
+ Value *ShadowPtr =
+ getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true)
+ .first;
IRB.CreateStore(getCleanShadow(Ty),
IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
@@ -2471,7 +2543,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
unsigned Alignment = 1;
Value *ShadowPtr, *OriginPtr;
std::tie(ShadowPtr, OriginPtr) =
- getShadowOriginPtr(Addr, IRB, Ty, Alignment);
+ getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
if (ClCheckAccessAddress)
insertShadowCheck(Addr, &I);
@@ -2482,11 +2554,98 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
insertShadowCheck(Shadow, Origin, &I);
}
+ void handleMaskedStore(IntrinsicInst &I) {
+ IRBuilder<> IRB(&I);
+ Value *V = I.getArgOperand(0);
+ Value *Addr = I.getArgOperand(1);
+ unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
+ Value *Mask = I.getArgOperand(3);
+ Value *Shadow = getShadow(V);
+
+ Value *ShadowPtr;
+ Value *OriginPtr;
+ std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
+ Addr, IRB, Shadow->getType(), Align, /*isStore*/ true);
+
+ if (ClCheckAccessAddress) {
+ insertShadowCheck(Addr, &I);
+ // Uninitialized mask is kind of like uninitialized address, but not as
+ // scary.
+ insertShadowCheck(Mask, &I);
+ }
+
+ IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask);
+
+ if (MS.TrackOrigins) {
+ auto &DL = F.getParent()->getDataLayout();
+ paintOrigin(IRB, getOrigin(V), OriginPtr,
+ DL.getTypeStoreSize(Shadow->getType()),
+ std::max(Align, kMinOriginAlignment));
+ }
+ }
+
+ bool handleMaskedLoad(IntrinsicInst &I) {
+ IRBuilder<> IRB(&I);
+ Value *Addr = I.getArgOperand(0);
+ unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
+ Value *Mask = I.getArgOperand(2);
+ Value *PassThru = I.getArgOperand(3);
+
+ Type *ShadowTy = getShadowTy(&I);
+ Value *ShadowPtr, *OriginPtr;
+ if (PropagateShadow) {
+ std::tie(ShadowPtr, OriginPtr) =
+ getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false);
+ setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask,
+ getShadow(PassThru), "_msmaskedld"));
+ } else {
+ setShadow(&I, getCleanShadow(&I));
+ }
+
+ if (ClCheckAccessAddress) {
+ insertShadowCheck(Addr, &I);
+ insertShadowCheck(Mask, &I);
+ }
+
+ if (MS.TrackOrigins) {
+ if (PropagateShadow) {
+ // Choose between PassThru's and the loaded value's origins.
+ Value *MaskedPassThruShadow = IRB.CreateAnd(
+ getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
+
+ Value *Acc = IRB.CreateExtractElement(
+ MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
+ for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
+ ++i) {
+ Value *More = IRB.CreateExtractElement(
+ MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
+ Acc = IRB.CreateOr(Acc, More);
+ }
+
+ Value *Origin = IRB.CreateSelect(
+ IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
+ getOrigin(PassThru), IRB.CreateLoad(OriginPtr));
+
+ setOrigin(&I, Origin);
+ } else {
+ setOrigin(&I, getCleanOrigin());
+ }
+ }
+ return true;
+ }
+
+
void visitIntrinsicInst(IntrinsicInst &I) {
switch (I.getIntrinsicID()) {
case Intrinsic::bswap:
handleBswap(I);
break;
+ case Intrinsic::masked_store:
+ handleMaskedStore(I);
+ break;
+ case Intrinsic::masked_load:
+ handleMaskedLoad(I);
+ break;
case Intrinsic::x86_sse_stmxcsr:
handleStmxcsr(I);
break;
@@ -2501,20 +2660,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
case Intrinsic::x86_avx512_cvttss2usi:
case Intrinsic::x86_avx512_cvttsd2usi64:
case Intrinsic::x86_avx512_cvttsd2usi:
- case Intrinsic::x86_avx512_cvtusi2sd:
case Intrinsic::x86_avx512_cvtusi2ss:
case Intrinsic::x86_avx512_cvtusi642sd:
case Intrinsic::x86_avx512_cvtusi642ss:
case Intrinsic::x86_sse2_cvtsd2si64:
case Intrinsic::x86_sse2_cvtsd2si:
case Intrinsic::x86_sse2_cvtsd2ss:
- case Intrinsic::x86_sse2_cvtsi2sd:
- case Intrinsic::x86_sse2_cvtsi642sd:
- case Intrinsic::x86_sse2_cvtss2sd:
case Intrinsic::x86_sse2_cvttsd2si64:
case Intrinsic::x86_sse2_cvttsd2si:
- case Intrinsic::x86_sse_cvtsi2ss:
- case Intrinsic::x86_sse_cvtsi642ss:
case Intrinsic::x86_sse_cvtss2si64:
case Intrinsic::x86_sse_cvtss2si:
case Intrinsic::x86_sse_cvttss2si64:
@@ -2715,7 +2868,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// outputs as clean. Note that any side effects of the inline asm that are
// not immediately visible in its constraints are not handled.
if (Call->isInlineAsm()) {
- visitInstruction(I);
+ if (ClHandleAsmConservative)
+ visitAsmInstruction(I);
+ else
+ visitInstruction(I);
return;
}
@@ -2738,13 +2894,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRBuilder<> IRB(&I);
unsigned ArgOffset = 0;
- DEBUG(dbgs() << " CallSite: " << I << "\n");
+ LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n");
for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
ArgIt != End; ++ArgIt) {
Value *A = *ArgIt;
unsigned i = ArgIt - CS.arg_begin();
if (!A->getType()->isSized()) {
- DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
+ LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
continue;
}
unsigned Size = 0;
@@ -2754,8 +2910,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// __msan_param_tls.
Value *ArgShadow = getShadow(A);
Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
- DEBUG(dbgs() << " Arg#" << i << ": " << *A <<
- " Shadow: " << *ArgShadow << "\n");
+ LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
+ << " Shadow: " << *ArgShadow << "\n");
bool ArgIsInitialized = false;
const DataLayout &DL = F.getParent()->getDataLayout();
if (CS.paramHasAttr(i, Attribute::ByVal)) {
@@ -2765,10 +2921,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (ArgOffset + Size > kParamTLSSize) break;
unsigned ParamAlignment = CS.getParamAlignment(i);
unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
- Value *AShadowPtr =
- getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment).first;
+ Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
+ Alignment, /*isStore*/ false)
+ .first;
- Store = IRB.CreateMemCpy(ArgShadowBase, AShadowPtr, Size, Alignment);
+ Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
+ Alignment, Size);
} else {
Size = DL.getTypeAllocSize(A->getType());
if (ArgOffset + Size > kParamTLSSize) break;
@@ -2782,10 +2940,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
getOriginPtrForArgument(A, IRB, ArgOffset));
(void)Store;
assert(Size != 0 && Store != nullptr);
- DEBUG(dbgs() << " Param:" << *Store << "\n");
+ LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
ArgOffset += alignTo(Size, 8);
}
- DEBUG(dbgs() << " done with call args\n");
+ LLVM_DEBUG(dbgs() << " done with call args\n");
FunctionType *FT =
cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
@@ -2888,8 +3046,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRB.CreateCall(MS.MsanPoisonStackFn,
{IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
} else {
- Value *ShadowBase =
- getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(), I.getAlignment()).first;
+ Value *ShadowBase = getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(),
+ I.getAlignment(), /*isStore*/ true)
+ .first;
Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
@@ -2991,24 +3150,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void visitExtractValueInst(ExtractValueInst &I) {
IRBuilder<> IRB(&I);
Value *Agg = I.getAggregateOperand();
- DEBUG(dbgs() << "ExtractValue: " << I << "\n");
+ LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
Value *AggShadow = getShadow(Agg);
- DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
+ LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
- DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
+ LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
setShadow(&I, ResShadow);
setOriginForNaryOp(I);
}
void visitInsertValueInst(InsertValueInst &I) {
IRBuilder<> IRB(&I);
- DEBUG(dbgs() << "InsertValue: " << I << "\n");
+ LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
Value *AggShadow = getShadow(I.getAggregateOperand());
Value *InsShadow = getShadow(I.getInsertedValueOperand());
- DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
- DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
+ LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
+ LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
- DEBUG(dbgs() << " Res: " << *Res << "\n");
+ LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
setShadow(&I, Res);
setOriginForNaryOp(I);
}
@@ -3023,25 +3182,58 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
void visitResumeInst(ResumeInst &I) {
- DEBUG(dbgs() << "Resume: " << I << "\n");
+ LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
// Nothing to do here.
}
void visitCleanupReturnInst(CleanupReturnInst &CRI) {
- DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
+ LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
// Nothing to do here.
}
void visitCatchReturnInst(CatchReturnInst &CRI) {
- DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
+ LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
// Nothing to do here.
}
+ void visitAsmInstruction(Instruction &I) {
+ // Conservative inline assembly handling: check for poisoned shadow of
+ // asm() arguments, then unpoison the result and all the memory locations
+ // pointed to by those arguments.
+ CallInst *CI = dyn_cast<CallInst>(&I);
+
+ for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
+ Value *Operand = CI->getOperand(i);
+ if (Operand->getType()->isSized())
+ insertShadowCheck(Operand, &I);
+ }
+ setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
+ IRBuilder<> IRB(&I);
+ IRB.SetInsertPoint(I.getNextNode());
+ for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
+ Value *Operand = CI->getOperand(i);
+ Type *OpType = Operand->getType();
+ if (!OpType->isPointerTy())
+ continue;
+ Type *ElType = OpType->getPointerElementType();
+ if (!ElType->isSized())
+ continue;
+ Value *ShadowPtr, *OriginPtr;
+ std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
+ Operand, IRB, ElType, /*Alignment*/ 1, /*isStore*/ true);
+ Value *CShadow = getCleanShadow(ElType);
+ IRB.CreateStore(
+ CShadow,
+ IRB.CreatePointerCast(ShadowPtr, CShadow->getType()->getPointerTo()));
+ }
+ }
+
void visitInstruction(Instruction &I) {
// Everything else: stop propagating and check for poisoned shadow.
if (ClDumpStrictInstructions)
dumpInst(I);
- DEBUG(dbgs() << "DEFAULT: " << I << "\n");
+ LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
Value *Operand = I.getOperand(i);
if (Operand->getType()->isSized())
@@ -3052,7 +3244,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
};
-/// \brief AMD64-specific implementation of VarArgHelper.
+/// AMD64-specific implementation of VarArgHelper.
struct VarArgAMD64Helper : public VarArgHelper {
// An unfortunate workaround for asymmetric lowering of va_arg stuff.
// See a comment in visitCallSite for more details.
@@ -3116,10 +3308,12 @@ struct VarArgAMD64Helper : public VarArgHelper {
getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
OverflowOffset += alignTo(ArgSize, 8);
Value *ShadowPtr, *OriginPtr;
- std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
- A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment);
+ std::tie(ShadowPtr, OriginPtr) =
+ MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
+ /*isStore*/ false);
- IRB.CreateMemCpy(ShadowBase, ShadowPtr, ArgSize, kShadowTLSAlignment);
+ IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
+ kShadowTLSAlignment, ArgSize);
} else {
ArgKind AK = classifyArgument(A);
if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
@@ -3157,7 +3351,7 @@ struct VarArgAMD64Helper : public VarArgHelper {
IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
}
- /// \brief Compute the shadow address for a given va_arg.
+ /// Compute the shadow address for a given va_arg.
Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
int ArgOffset) {
Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
@@ -3172,7 +3366,8 @@ struct VarArgAMD64Helper : public VarArgHelper {
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
+ /*isStore*/ true);
// Unpoison the whole __va_list_tag.
// FIXME: magic ABI constants.
@@ -3200,13 +3395,13 @@ struct VarArgAMD64Helper : public VarArgHelper {
if (!VAStartInstrumentationList.empty()) {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
- IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+ IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
Value *CopySize =
IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
VAArgOverflowSize);
VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
- IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
+ IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
}
// Instrument va_start.
@@ -3219,33 +3414,33 @@ struct VarArgAMD64Helper : public VarArgHelper {
Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
ConstantInt::get(MS.IntptrTy, 16)),
- Type::getInt64PtrTy(*MS.C));
+ PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
unsigned Alignment = 16;
std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
- Alignment);
- IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, AMD64FpEndOffset,
- Alignment);
+ Alignment, /*isStore*/ true);
+ IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
+ AMD64FpEndOffset);
Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
ConstantInt::get(MS.IntptrTy, 8)),
- Type::getInt64PtrTy(*MS.C));
+ PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
- Alignment);
+ Alignment, /*isStore*/ true);
Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
AMD64FpEndOffset);
- IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize,
- Alignment);
+ IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
+ VAArgOverflowSize);
}
}
};
-/// \brief MIPS64-specific implementation of VarArgHelper.
+/// MIPS64-specific implementation of VarArgHelper.
struct VarArgMIPS64Helper : public VarArgHelper {
Function &F;
MemorySanitizer &MS;
@@ -3286,7 +3481,7 @@ struct VarArgMIPS64Helper : public VarArgHelper {
IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
}
- /// \brief Compute the shadow address for a given va_arg.
+ /// Compute the shadow address for a given va_arg.
Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
int ArgOffset) {
Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
@@ -3301,8 +3496,8 @@ struct VarArgMIPS64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
/* size */ 8, Alignment, false);
}
@@ -3313,8 +3508,8 @@ struct VarArgMIPS64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
/* size */ 8, Alignment, false);
}
@@ -3322,7 +3517,7 @@ struct VarArgMIPS64Helper : public VarArgHelper {
void finalizeInstrumentation() override {
assert(!VAArgSize && !VAArgTLSCopy &&
"finalizeInstrumentation called twice");
- IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+ IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
VAArgSize);
@@ -3331,7 +3526,7 @@ struct VarArgMIPS64Helper : public VarArgHelper {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
- IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
+ IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
}
// Instrument va_start.
@@ -3341,20 +3536,21 @@ struct VarArgMIPS64Helper : public VarArgHelper {
IRBuilder<> IRB(OrigInst->getNextNode());
Value *VAListTag = OrigInst->getArgOperand(0);
Value *RegSaveAreaPtrPtr =
- IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
- Type::getInt64PtrTy(*MS.C));
+ IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
+ PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
unsigned Alignment = 8;
std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
- Alignment);
- IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, Alignment);
+ Alignment, /*isStore*/ true);
+ IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
+ CopySize);
}
}
};
-/// \brief AArch64-specific implementation of VarArgHelper.
+/// AArch64-specific implementation of VarArgHelper.
struct VarArgAArch64Helper : public VarArgHelper {
static const unsigned kAArch64GrArgSize = 64;
static const unsigned kAArch64VrArgSize = 128;
@@ -3461,8 +3657,8 @@ struct VarArgAArch64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
/* size */ 32, Alignment, false);
}
@@ -3473,8 +3669,8 @@ struct VarArgAArch64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
/* size */ 32, Alignment, false);
}
@@ -3506,13 +3702,13 @@ struct VarArgAArch64Helper : public VarArgHelper {
if (!VAStartInstrumentationList.empty()) {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
- IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+ IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
Value *CopySize =
IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
VAArgOverflowSize);
VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
- IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
+ IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
}
Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
@@ -3563,14 +3759,14 @@ struct VarArgAArch64Helper : public VarArgHelper {
Value *GrRegSaveAreaShadowPtr =
MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
- /*Alignment*/ 8)
+ /*Alignment*/ 8, /*isStore*/ true)
.first;
Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
GrRegSaveAreaShadowPtrOff);
Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
- IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, GrSrcPtr, GrCopySize, 8);
+ IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize);
// Again, but for FP/SIMD values.
Value *VrRegSaveAreaShadowPtrOff =
@@ -3578,7 +3774,7 @@ struct VarArgAArch64Helper : public VarArgHelper {
Value *VrRegSaveAreaShadowPtr =
MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
- /*Alignment*/ 8)
+ /*Alignment*/ 8, /*isStore*/ true)
.first;
Value *VrSrcPtr = IRB.CreateInBoundsGEP(
@@ -3588,25 +3784,25 @@ struct VarArgAArch64Helper : public VarArgHelper {
VrRegSaveAreaShadowPtrOff);
Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
- IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, VrSrcPtr, VrCopySize, 8);
+ IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize);
// And finally for remaining arguments.
Value *StackSaveAreaShadowPtr =
MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
- /*Alignment*/ 16)
+ /*Alignment*/ 16, /*isStore*/ true)
.first;
Value *StackSrcPtr =
IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
IRB.getInt32(AArch64VAEndOffset));
- IRB.CreateMemCpy(StackSaveAreaShadowPtr, StackSrcPtr,
- VAArgOverflowSize, 16);
+ IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16,
+ VAArgOverflowSize);
}
}
};
-/// \brief PowerPC64-specific implementation of VarArgHelper.
+/// PowerPC64-specific implementation of VarArgHelper.
struct VarArgPowerPC64Helper : public VarArgHelper {
Function &F;
MemorySanitizer &MS;
@@ -3657,9 +3853,10 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
VAArgOffset - VAArgBase);
Value *AShadowPtr, *AOriginPtr;
std::tie(AShadowPtr, AOriginPtr) = MSV.getShadowOriginPtr(
- A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment);
+ A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment, /*isStore*/ false);
- IRB.CreateMemCpy(Base, AShadowPtr, ArgSize, kShadowTLSAlignment);
+ IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
+ kShadowTLSAlignment, ArgSize);
}
VAArgOffset += alignTo(ArgSize, 8);
} else {
@@ -3704,7 +3901,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
}
- /// \brief Compute the shadow address for a given va_arg.
+ /// Compute the shadow address for a given va_arg.
Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
int ArgOffset) {
Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
@@ -3719,8 +3916,8 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
/* size */ 8, Alignment, false);
}
@@ -3730,8 +3927,8 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
Value *VAListTag = I.getArgOperand(0);
Value *ShadowPtr, *OriginPtr;
unsigned Alignment = 8;
- std::tie(ShadowPtr, OriginPtr) =
- MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment);
+ std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
+ VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
// Unpoison the whole __va_list_tag.
// FIXME: magic ABI constants.
IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
@@ -3741,7 +3938,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
void finalizeInstrumentation() override {
assert(!VAArgSize && !VAArgTLSCopy &&
"finalizeInstrumentation called twice");
- IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
+ IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
VAArgSize);
@@ -3750,7 +3947,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
- IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
+ IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
}
// Instrument va_start.
@@ -3760,20 +3957,21 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
IRBuilder<> IRB(OrigInst->getNextNode());
Value *VAListTag = OrigInst->getArgOperand(0);
Value *RegSaveAreaPtrPtr =
- IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
- Type::getInt64PtrTy(*MS.C));
+ IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
+ PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
unsigned Alignment = 8;
std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
- Alignment);
- IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, Alignment);
+ Alignment, /*isStore*/ true);
+ IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
+ CopySize);
}
}
};
-/// \brief A no-op implementation of VarArgHelper.
+/// A no-op implementation of VarArgHelper.
struct VarArgNoOpHelper : public VarArgHelper {
VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
MemorySanitizerVisitor &MSV) {}
@@ -3796,8 +3994,7 @@ static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
Triple TargetTriple(Func.getParent()->getTargetTriple());
if (TargetTriple.getArch() == Triple::x86_64)
return new VarArgAMD64Helper(Func, Msan, Visitor);
- else if (TargetTriple.getArch() == Triple::mips64 ||
- TargetTriple.getArch() == Triple::mips64el)
+ else if (TargetTriple.isMIPS64())
return new VarArgMIPS64Helper(Func, Msan, Visitor);
else if (TargetTriple.getArch() == Triple::aarch64)
return new VarArgAArch64Helper(Func, Msan, Visitor);
diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index cb4b3a9c2545..307b7eaa2196 100644
--- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -48,7 +48,7 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/PGOInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "CFGMST.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
@@ -119,6 +119,7 @@
#include <vector>
using namespace llvm;
+using ProfileCount = Function::ProfileCount;
#define DEBUG_TYPE "pgo-instrumentation"
@@ -223,8 +224,8 @@ static cl::opt<bool>
EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden,
cl::desc("When this option is on, the annotated "
"branch probability will be emitted as "
- " optimization remarks: -Rpass-analysis="
- "pgo-instr-use"));
+ "optimization remarks: -{Rpass|"
+ "pass-remarks}=pgo-instrumentation"));
// Command line option to turn on CFG dot dump after profile annotation.
// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
@@ -448,7 +449,7 @@ ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
namespace {
-/// \brief An MST based instrumentation for PGO
+/// An MST based instrumentation for PGO
///
/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
/// in the function level.
@@ -545,7 +546,7 @@ public:
computeCFGHash();
if (!ComdatMembers.empty())
renameComdatFunction();
- DEBUG(dumpInfo("after CFGMST"));
+ LLVM_DEBUG(dumpInfo("after CFGMST"));
NumOfPGOBB += MST.BBInfos.size();
for (auto &E : MST.AllEdges) {
@@ -595,12 +596,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
(uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
(uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
- DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"
- << " CRC = " << JC.getCRC()
- << ", Selects = " << SIVisitor.getNumOfSelectInsts()
- << ", Edges = " << MST.AllEdges.size()
- << ", ICSites = " << ValueSites[IPVK_IndirectCallTarget].size()
- << ", Hash = " << FunctionHash << "\n";);
+ LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"
+ << " CRC = " << JC.getCRC()
+ << ", Selects = " << SIVisitor.getNumOfSelectInsts()
+ << ", Edges = " << MST.AllEdges.size() << ", ICSites = "
+ << ValueSites[IPVK_IndirectCallTarget].size()
+ << ", Hash = " << FunctionHash << "\n";);
}
// Check if we can safely rename this Comdat function.
@@ -701,8 +702,8 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
// For a critical edge, we have to split. Instrument the newly
// created BB.
NumOfPGOSplit++;
- DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> "
- << getBBInfo(DestBB).Index << "\n");
+ LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index
+ << " --> " << getBBInfo(DestBB).Index << "\n");
unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum);
assert(InstrBB && "Critical edge is not split");
@@ -752,8 +753,8 @@ static void instrumentOneFunc(
for (auto &I : FuncInfo.ValueSites[IPVK_IndirectCallTarget]) {
CallSite CS(I);
Value *Callee = CS.getCalledValue();
- DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "
- << NumIndirectCallSites << "\n");
+ LLVM_DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "
+ << NumIndirectCallSites << "\n");
IRBuilder<> Builder(I);
assert(Builder.GetInsertPoint() != I->getParent()->end() &&
"Cannot get the Instrumentation point");
@@ -861,7 +862,7 @@ public:
// Set the branch weights based on the count values.
void setBranchWeights();
- // Annotate the value profile call sites all all value kind.
+ // Annotate the value profile call sites for all value kind.
void annotateValueSites();
// Annotate the value profile call sites for one value kind.
@@ -1041,14 +1042,14 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
NumOfPGOFunc++;
- DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
+ LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
uint64_t ValueSum = 0;
for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
- DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");
+ LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");
ValueSum += CountFromProfile[I];
}
- DEBUG(dbgs() << "SUM = " << ValueSum << "\n");
+ LLVM_DEBUG(dbgs() << "SUM = " << ValueSum << "\n");
getBBInfo(nullptr).UnknownCountOutEdge = 2;
getBBInfo(nullptr).UnknownCountInEdge = 2;
@@ -1128,7 +1129,7 @@ void PGOUseFunc::populateCounters() {
}
}
- DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");
+ LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");
#ifndef NDEBUG
// Assert every BB has a valid counter.
for (auto &BB : F) {
@@ -1139,7 +1140,7 @@ void PGOUseFunc::populateCounters() {
}
#endif
uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
- F.setEntryCount(FuncEntryCount);
+ F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
uint64_t FuncMaxCount = FuncEntryCount;
for (auto &BB : F) {
auto BI = findBBInfo(&BB);
@@ -1153,13 +1154,13 @@ void PGOUseFunc::populateCounters() {
FuncInfo.SIVisitor.annotateSelects(F, this, &CountPosition);
assert(CountPosition == ProfileCountSize);
- DEBUG(FuncInfo.dumpInfo("after reading profile."));
+ LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."));
}
// Assign the scaled count values to the BB with multiple out edges.
void PGOUseFunc::setBranchWeights() {
// Generate MD_prof metadata for every branch instruction.
- DEBUG(dbgs() << "\nSetting branch weights.\n");
+ LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n");
for (auto &BB : F) {
TerminatorInst *TI = BB.getTerminator();
if (TI->getNumSuccessors() < 2)
@@ -1200,7 +1201,7 @@ static bool isIndirectBrTarget(BasicBlock *BB) {
}
void PGOUseFunc::annotateIrrLoopHeaderWeights() {
- DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n");
+ LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n");
// Find irr loop headers
for (auto &BB : F) {
// As a heuristic also annotate indrectbr targets as they have a high chance
@@ -1333,9 +1334,9 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) {
}
for (auto &I : ValueSites) {
- DEBUG(dbgs() << "Read one value site profile (kind = " << Kind
- << "): Index = " << ValueSiteIndex << " out of "
- << NumValueSites << "\n");
+ LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kind
+ << "): Index = " << ValueSiteIndex << " out of "
+ << NumValueSites << "\n");
annotateValueSite(*M, *I, ProfileRecord,
static_cast<InstrProfValueKind>(Kind), ValueSiteIndex,
Kind == IPVK_MemOPSize ? MaxNumMemOPAnnotations
@@ -1431,7 +1432,7 @@ static bool annotateAllFunctions(
Module &M, StringRef ProfileFileName,
function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
- DEBUG(dbgs() << "Read in profile counters: ");
+ LLVM_DEBUG(dbgs() << "Read in profile counters: ");
auto &Ctx = M.getContext();
// Read the counter array from file.
auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName);
@@ -1517,12 +1518,13 @@ static bool annotateAllFunctions(
// inconsistent MST between prof-gen and prof-use.
for (auto &F : HotFunctions) {
F->addFnAttr(Attribute::InlineHint);
- DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()
- << "\n");
+ LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()
+ << "\n");
}
for (auto &F : ColdFunctions) {
F->addFnAttr(Attribute::Cold);
- DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n");
+ LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()
+ << "\n");
}
return true;
}
@@ -1585,22 +1587,25 @@ void llvm::setProfMetadata(Module *M, Instruction *TI,
for (const auto &ECI : EdgeCounts)
Weights.push_back(scaleBranchCount(ECI, Scale));
- DEBUG(dbgs() << "Weight is: ";
- for (const auto &W : Weights) { dbgs() << W << " "; }
- dbgs() << "\n";);
+ LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &W
+ : Weights) {
+ dbgs() << W << " ";
+ } dbgs() << "\n";);
TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
if (EmitBranchProbability) {
std::string BrCondStr = getBranchCondString(TI);
if (BrCondStr.empty())
return;
- unsigned WSum =
- std::accumulate(Weights.begin(), Weights.end(), 0,
- [](unsigned w1, unsigned w2) { return w1 + w2; });
+ uint64_t WSum =
+ std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0,
+ [](uint64_t w1, uint64_t w2) { return w1 + w2; });
uint64_t TotalCount =
- std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), 0,
+ std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,
[](uint64_t c1, uint64_t c2) { return c1 + c2; });
- BranchProbability BP(Weights[0], WSum);
+ Scale = calculateCountScale(WSum);
+ BranchProbability BP(scaleBranchCount(Weights[0], Scale),
+ scaleBranchCount(WSum, Scale));
std::string BranchProbStr;
raw_string_ostream OS(BranchProbStr);
OS << BP;
diff --git a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 95eb3680403a..2c71e75dadcc 100644
--- a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -25,6 +25,8 @@
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/DomTreeUpdater.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
@@ -44,7 +46,7 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/Transforms/PGOInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <cassert>
#include <cstdint>
@@ -112,6 +114,7 @@ private:
AU.addRequired<BlockFrequencyInfoWrapperPass>();
AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
AU.addPreserved<GlobalsAAWrapperPass>();
+ AU.addPreserved<DominatorTreeWrapperPass>();
}
};
} // end anonymous namespace
@@ -133,8 +136,8 @@ namespace {
class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
public:
MemOPSizeOpt(Function &Func, BlockFrequencyInfo &BFI,
- OptimizationRemarkEmitter &ORE)
- : Func(Func), BFI(BFI), ORE(ORE), Changed(false) {
+ OptimizationRemarkEmitter &ORE, DominatorTree *DT)
+ : Func(Func), BFI(BFI), ORE(ORE), DT(DT), Changed(false) {
ValueDataArray =
llvm::make_unique<InstrProfValueData[]>(MemOPMaxVersion + 2);
// Get the MemOPSize range information from option MemOPSizeRange,
@@ -151,8 +154,9 @@ public:
if (perform(MI)) {
Changed = true;
++NumOfPGOMemOPOpt;
- DEBUG(dbgs() << "MemOP call: " << MI->getCalledFunction()->getName()
- << "is Transformed.\n");
+ LLVM_DEBUG(dbgs() << "MemOP call: "
+ << MI->getCalledFunction()->getName()
+ << "is Transformed.\n");
}
}
}
@@ -169,6 +173,7 @@ private:
Function &Func;
BlockFrequencyInfo &BFI;
OptimizationRemarkEmitter &ORE;
+ DominatorTree *DT;
bool Changed;
std::vector<MemIntrinsic *> WorkList;
// Start of the previse range.
@@ -245,9 +250,9 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
}
ArrayRef<InstrProfValueData> VDs(ValueDataArray.get(), NumVals);
- DEBUG(dbgs() << "Read one memory intrinsic profile with count " << ActualCount
- << "\n");
- DEBUG(
+ LLVM_DEBUG(dbgs() << "Read one memory intrinsic profile with count "
+ << ActualCount << "\n");
+ LLVM_DEBUG(
for (auto &VD
: VDs) { dbgs() << " (" << VD.Value << "," << VD.Count << ")\n"; });
@@ -260,8 +265,8 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
TotalCount = ActualCount;
if (MemOPScaleCount)
- DEBUG(dbgs() << "Scale counts: numerator = " << ActualCount
- << " denominator = " << SavedTotalCount << "\n");
+ LLVM_DEBUG(dbgs() << "Scale counts: numerator = " << ActualCount
+ << " denominator = " << SavedTotalCount << "\n");
// Keeping track of the count of the default case:
uint64_t RemainCount = TotalCount;
@@ -310,9 +315,9 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
uint64_t SumForOpt = TotalCount - RemainCount;
- DEBUG(dbgs() << "Optimize one memory intrinsic call to " << Version
- << " Versions (covering " << SumForOpt << " out of "
- << TotalCount << ")\n");
+ LLVM_DEBUG(dbgs() << "Optimize one memory intrinsic call to " << Version
+ << " Versions (covering " << SumForOpt << " out of "
+ << TotalCount << ")\n");
// mem_op(..., size)
// ==>
@@ -331,19 +336,20 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
// merge_bb:
BasicBlock *BB = MI->getParent();
- DEBUG(dbgs() << "\n\n== Basic Block Before ==\n");
- DEBUG(dbgs() << *BB << "\n");
+ LLVM_DEBUG(dbgs() << "\n\n== Basic Block Before ==\n");
+ LLVM_DEBUG(dbgs() << *BB << "\n");
auto OrigBBFreq = BFI.getBlockFreq(BB);
- BasicBlock *DefaultBB = SplitBlock(BB, MI);
+ BasicBlock *DefaultBB = SplitBlock(BB, MI, DT);
BasicBlock::iterator It(*MI);
++It;
assert(It != DefaultBB->end());
- BasicBlock *MergeBB = SplitBlock(DefaultBB, &(*It));
+ BasicBlock *MergeBB = SplitBlock(DefaultBB, &(*It), DT);
MergeBB->setName("MemOP.Merge");
BFI.setBlockFreq(MergeBB, OrigBBFreq.getFrequency());
DefaultBB->setName("MemOP.Default");
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
auto &Ctx = Func.getContext();
IRBuilder<> IRB(BB);
BB->getTerminator()->eraseFromParent();
@@ -358,7 +364,11 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
annotateValueSite(*Func.getParent(), *MI, VDs.slice(Version),
SavedRemainCount, IPVK_MemOPSize, NumVals);
- DEBUG(dbgs() << "\n\n== Basic Block After==\n");
+ LLVM_DEBUG(dbgs() << "\n\n== Basic Block After==\n");
+
+ std::vector<DominatorTree::UpdateType> Updates;
+ if (DT)
+ Updates.reserve(2 * SizeIds.size());
for (uint64_t SizeId : SizeIds) {
BasicBlock *CaseBB = BasicBlock::Create(
@@ -374,13 +384,20 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
IRBuilder<> IRBCase(CaseBB);
IRBCase.CreateBr(MergeBB);
SI->addCase(CaseSizeId, CaseBB);
- DEBUG(dbgs() << *CaseBB << "\n");
+ if (DT) {
+ Updates.push_back({DominatorTree::Insert, CaseBB, MergeBB});
+ Updates.push_back({DominatorTree::Insert, BB, CaseBB});
+ }
+ LLVM_DEBUG(dbgs() << *CaseBB << "\n");
}
+ DTU.applyUpdates(Updates);
+ Updates.clear();
+
setProfMetadata(Func.getParent(), SI, CaseCounts, MaxCount);
- DEBUG(dbgs() << *BB << "\n");
- DEBUG(dbgs() << *DefaultBB << "\n");
- DEBUG(dbgs() << *MergeBB << "\n");
+ LLVM_DEBUG(dbgs() << *BB << "\n");
+ LLVM_DEBUG(dbgs() << *DefaultBB << "\n");
+ LLVM_DEBUG(dbgs() << *MergeBB << "\n");
ORE.emit([&]() {
using namespace ore;
@@ -396,13 +413,14 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
} // namespace
static bool PGOMemOPSizeOptImpl(Function &F, BlockFrequencyInfo &BFI,
- OptimizationRemarkEmitter &ORE) {
+ OptimizationRemarkEmitter &ORE,
+ DominatorTree *DT) {
if (DisableMemOPOPT)
return false;
if (F.hasFnAttribute(Attribute::OptimizeForSize))
return false;
- MemOPSizeOpt MemOPSizeOpt(F, BFI, ORE);
+ MemOPSizeOpt MemOPSizeOpt(F, BFI, ORE, DT);
MemOPSizeOpt.perform();
return MemOPSizeOpt.isChanged();
}
@@ -411,7 +429,9 @@ bool PGOMemOPSizeOptLegacyPass::runOnFunction(Function &F) {
BlockFrequencyInfo &BFI =
getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
- return PGOMemOPSizeOptImpl(F, BFI, ORE);
+ auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
+ DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
+ return PGOMemOPSizeOptImpl(F, BFI, ORE, DT);
}
namespace llvm {
@@ -421,11 +441,13 @@ PreservedAnalyses PGOMemOPSizeOpt::run(Function &F,
FunctionAnalysisManager &FAM) {
auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
- bool Changed = PGOMemOPSizeOptImpl(F, BFI, ORE);
+ auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+ bool Changed = PGOMemOPSizeOptImpl(F, BFI, ORE, DT);
if (!Changed)
return PreservedAnalyses::all();
auto PA = PreservedAnalyses();
PA.preserve<GlobalsAA>();
+ PA.preserve<DominatorTreeAnalysis>();
return PA;
}
} // namespace llvm
diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index d950e2e730f2..a4dd48c8dd6a 100644
--- a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -35,7 +35,6 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -243,6 +242,7 @@ private:
GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
GlobalVariable *FunctionPCsArray; // for pc-table.
SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
+ SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
SanitizerCoverageOptions Options;
};
@@ -405,6 +405,7 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
// so we need to prevent them from being dead stripped.
if (TargetTriple.isOSBinFormatMachO())
appendToUsed(M, GlobalsToAppendToUsed);
+ appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed);
return true;
}
@@ -480,6 +481,8 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) {
if (F.getName() == "__local_stdio_printf_options" ||
F.getName() == "__local_stdio_scanf_options")
return false;
+ if (isa<UnreachableInst>(F.getEntryBlock().getTerminator()))
+ return false;
// Don't instrument functions using SEH for now. Splitting basic blocks like
// we do for coverage breaks WinEHPrepare.
// FIXME: Remove this when SEH no longer uses landingpad pattern matching.
@@ -592,11 +595,15 @@ void SanitizerCoverageModule::CreateFunctionLocalArrays(
if (Options.Inline8bitCounters) {
Function8bitCounterArray = CreateFunctionLocalArrayInSection(
AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName);
- GlobalsToAppendToUsed.push_back(Function8bitCounterArray);
+ GlobalsToAppendToCompilerUsed.push_back(Function8bitCounterArray);
+ MDNode *MD = MDNode::get(F.getContext(), ValueAsMetadata::get(&F));
+ Function8bitCounterArray->addMetadata(LLVMContext::MD_associated, *MD);
}
if (Options.PCTable) {
FunctionPCsArray = CreatePCArray(F, AllBlocks);
- GlobalsToAppendToUsed.push_back(FunctionPCsArray);
+ GlobalsToAppendToCompilerUsed.push_back(FunctionPCsArray);
+ MDNode *MD = MDNode::get(F.getContext(), ValueAsMetadata::get(&F));
+ FunctionPCsArray->addMetadata(LLVMContext::MD_associated, *MD);
}
}
@@ -659,11 +666,11 @@ void SanitizerCoverageModule::InjectTraceForSwitch(
C = ConstantExpr::getCast(CastInst::ZExt, It.getCaseValue(), Int64Ty);
Initializers.push_back(C);
}
- std::sort(Initializers.begin() + 2, Initializers.end(),
- [](const Constant *A, const Constant *B) {
- return cast<ConstantInt>(A)->getLimitedValue() <
- cast<ConstantInt>(B)->getLimitedValue();
- });
+ llvm::sort(Initializers.begin() + 2, Initializers.end(),
+ [](const Constant *A, const Constant *B) {
+ return cast<ConstantInt>(A)->getLimitedValue() <
+ cast<ConstantInt>(B)->getLimitedValue();
+ });
ArrayType *ArrayOfInt64Ty = ArrayType::get(Int64Ty, Initializers.size());
GlobalVariable *GV = new GlobalVariable(
*CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index ec6904486e10..fa1e5a157a0f 100644
--- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -19,13 +19,14 @@
// The rest is handled by the run-time library.
//===----------------------------------------------------------------------===//
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Analysis/CaptureTracking.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Function.h"
@@ -44,7 +45,6 @@
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/EscapeEnumerator.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
@@ -339,7 +339,7 @@ bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) {
void ThreadSanitizer::chooseInstructionsToInstrument(
SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All,
const DataLayout &DL) {
- SmallSet<Value*, 8> WriteTargets;
+ SmallPtrSet<Value*, 8> WriteTargets;
// Iterate from the end.
for (Instruction *I : reverse(Local)) {
if (StoreInst *Store = dyn_cast<StoreInst>(I)) {
@@ -502,7 +502,7 @@ bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I,
if (Idx < 0)
return false;
if (IsWrite && isVtableAccess(I)) {
- DEBUG(dbgs() << " VPTR : " << *I << "\n");
+ LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n");
Value *StoredValue = cast<StoreInst>(I)->getValueOperand();
// StoredValue may be a vector type if we are storing several vptrs at once.
// In this case, just take the first element of the vector since this is
generated by cgit v1.3 (git 2.53.0) at 2026-05-13 02:07:31 +0000