diff options
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp')
-rw-r--r-- | llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp | 111 |
1 files changed, 87 insertions, 24 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index cc96bdd1d516..dcfc28887a48 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -52,6 +52,7 @@ #include "ValueProfileCollector.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -63,13 +64,13 @@ #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" @@ -376,6 +377,7 @@ private: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<BlockFrequencyInfoWrapperPass>(); + AU.addRequired<TargetLibraryInfoWrapperPass>(); } }; @@ -404,6 +406,7 @@ private: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<ProfileSummaryInfoWrapperPass>(); AU.addRequired<BlockFrequencyInfoWrapperPass>(); + AU.addRequired<TargetLibraryInfoWrapperPass>(); } }; @@ -436,6 +439,7 @@ INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) @@ -467,7 +471,7 @@ INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass, ModulePass * llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) { - return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName); + return new PGOInstrumentationGenCreateVarLegacyPass(std::string(CSInstrName)); } namespace { @@ -565,11 +569,11 @@ public: } FuncPGOInstrumentation( - Function &Func, + Function &Func, TargetLibraryInfo &TLI, std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr, bool IsCS = false) - : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func), + : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI), ValueSites(IPVK_Last + 1), SIVisitor(Func), MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); @@ -799,18 +803,50 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) { return canInstrument(InstrBB); } +// When generating value profiling calls on Windows routines that make use of +// handler funclets for exception processing an operand bundle needs to attached +// to the called function. This routine will set \p OpBundles to contain the +// funclet information, if any is needed, that should be placed on the generated +// value profiling call for the value profile candidate call. +static void +populateEHOperandBundle(VPCandidateInfo &Cand, + DenseMap<BasicBlock *, ColorVector> &BlockColors, + SmallVectorImpl<OperandBundleDef> &OpBundles) { + auto *OrigCall = dyn_cast<CallBase>(Cand.AnnotatedInst); + if (OrigCall && !isa<IntrinsicInst>(OrigCall)) { + // The instrumentation call should belong to the same funclet as a + // non-intrinsic call, so just copy the operand bundle, if any exists. + Optional<OperandBundleUse> ParentFunclet = + OrigCall->getOperandBundle(LLVMContext::OB_funclet); + if (ParentFunclet) + OpBundles.emplace_back(OperandBundleDef(*ParentFunclet)); + } else { + // Intrinsics or other instructions do not get funclet information from the + // front-end. Need to use the BlockColors that was computed by the routine + // colorEHFunclets to determine whether a funclet is needed. + if (!BlockColors.empty()) { + const ColorVector &CV = BlockColors.find(OrigCall->getParent())->second; + assert(CV.size() == 1 && "non-unique color for block!"); + Instruction *EHPad = CV.front()->getFirstNonPHI(); + if (EHPad->isEHPad()) + OpBundles.emplace_back("funclet", EHPad); + } + } +} + // Visit all edge and instrument the edges not in MST, and do value profiling. // Critical edges will be split. static void instrumentOneFunc( - Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, + Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI, + BlockFrequencyInfo *BFI, std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, bool IsCS) { // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI, - BFI, IsCS); + FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, TLI, ComdatMembers, true, + BPI, BFI, IsCS); std::vector<BasicBlock *> InstrumentBBs; FuncInfo.getInstrumentBBs(InstrumentBBs); unsigned NumCounters = @@ -839,6 +875,15 @@ static void instrumentOneFunc( NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size(); + // Intrinsic function calls do not have funclet operand bundles needed for + // Windows exception handling attached to them. However, if value profiling is + // inserted for one of these calls, then a funclet value will need to be set + // on the instrumentation call based on the funclet coloring. + DenseMap<BasicBlock *, ColorVector> BlockColors; + if (F.hasPersonalityFn() && + isFuncletEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) + BlockColors = colorEHFunclets(F); + // For each VP Kind, walk the VP candidates and instrument each one. for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { unsigned SiteIndex = 0; @@ -860,11 +905,14 @@ static void instrumentOneFunc( ToProfile = Builder.CreatePtrToInt(Cand.V, Builder.getInt64Ty()); assert(ToProfile && "value profiling Value is of unexpected type"); + SmallVector<OperandBundleDef, 1> OpBundles; + populateEHOperandBundle(Cand, BlockColors, OpBundles); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), ToProfile, - Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)}); + Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)}, + OpBundles); } } // IPVK_First <= Kind <= IPVK_Last } @@ -953,12 +1001,12 @@ namespace { class PGOUseFunc { public: - PGOUseFunc(Function &Func, Module *Modu, + PGOUseFunc(Function &Func, Module *Modu, TargetLibraryInfo &TLI, std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin, ProfileSummaryInfo *PSI, bool IsCS) : F(Func), M(Modu), BFI(BFIin), PSI(PSI), - FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS), + FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS), FreqAttr(FFA_Normal), IsCS(IsCS) {} // Read counts for the instrumented BB from profile. @@ -1295,7 +1343,7 @@ void PGOUseFunc::setBranchWeights() { if (TI->getNumSuccessors() < 2) continue; if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) || - isa<IndirectBrInst>(TI))) + isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI))) continue; if (getBBInfo(&BB).CountValue == 0) @@ -1460,7 +1508,8 @@ static void collectComdatMembers( } static bool InstrumentAllFunctions( - Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, + Module &M, function_ref<TargetLibraryInfo &(Function &)> LookupTLI, + function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) { // For the context-sensitve instrumentation, we should have a separated pass // (before LTO/ThinLTO linking) to create these variables. @@ -1472,9 +1521,10 @@ static bool InstrumentAllFunctions( for (auto &F : M) { if (F.isDeclaration()) continue; + auto &TLI = LookupTLI(F); auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); - instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS); + instrumentOneFunc(F, &M, TLI, BPI, BFI, ComdatMembers, IsCS); } return true; } @@ -1490,27 +1540,32 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { if (skipModule(M)) return false; + auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & { + return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); + }; auto LookupBPI = [this](Function &F) { return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI(); }; auto LookupBFI = [this](Function &F) { return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); }; - return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS); + return InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI, IsCS); } PreservedAnalyses PGOInstrumentationGen::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & { + return FAM.getResult<TargetLibraryAnalysis>(F); + }; auto LookupBPI = [&FAM](Function &F) { return &FAM.getResult<BranchProbabilityAnalysis>(F); }; - auto LookupBFI = [&FAM](Function &F) { return &FAM.getResult<BlockFrequencyAnalysis>(F); }; - if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS)) + if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1518,6 +1573,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M, static bool annotateAllFunctions( Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, + function_ref<TargetLibraryInfo &(Function &)> LookupTLI, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS) { @@ -1557,6 +1613,7 @@ static bool annotateAllFunctions( M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), IsCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr); + PSI->refresh(); std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers; collectComdatMembers(M, ComdatMembers); @@ -1565,12 +1622,13 @@ static bool annotateAllFunctions( for (auto &F : M) { if (F.isDeclaration()) continue; + auto &TLI = LookupTLI(F); auto *BPI = LookupBPI(F); auto *BFI = LookupBFI(F); // Split indirectbr critical edges here before computing the MST rather than // later in getInstrBB() to avoid invalidating it. SplitIndirectBrCriticalEdges(F, BPI, BFI); - PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, PSI, IsCS); + PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS); bool AllZeros = false; if (!Func.readCounters(PGOReader.get(), AllZeros)) continue; @@ -1651,10 +1709,12 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & { + return FAM.getResult<TargetLibraryAnalysis>(F); + }; auto LookupBPI = [&FAM](Function &F) { return &FAM.getResult<BranchProbabilityAnalysis>(F); }; - auto LookupBFI = [&FAM](Function &F) { return &FAM.getResult<BlockFrequencyAnalysis>(F); }; @@ -1662,7 +1722,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M, auto *PSI = &AM.getResult<ProfileSummaryAnalysis>(M); if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, - LookupBPI, LookupBFI, PSI, IsCS)) + LookupTLI, LookupBPI, LookupBFI, PSI, IsCS)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -1672,6 +1732,9 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { if (skipModule(M)) return false; + auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & { + return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); + }; auto LookupBPI = [this](Function &F) { return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI(); }; @@ -1680,13 +1743,13 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { }; auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); - return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, PSI, - IsCS); + return annotateAllFunctions(M, ProfileFileName, "", LookupTLI, LookupBPI, + LookupBFI, PSI, IsCS); } static std::string getSimpleNodeName(const BasicBlock *Node) { if (!Node->getName().empty()) - return Node->getName(); + return std::string(Node->getName()); std::string SimpleNodeName; raw_string_ostream OS(SimpleNodeName); @@ -1750,7 +1813,7 @@ void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) { template <> struct GraphTraits<PGOUseFunc *> { using NodeRef = const BasicBlock *; - using ChildIteratorType = succ_const_iterator; + using ChildIteratorType = const_succ_iterator; using nodes_iterator = pointer_iterator<Function::const_iterator>; static NodeRef getEntryNode(const PGOUseFunc *G) { @@ -1777,7 +1840,7 @@ template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits { : DefaultDOTGraphTraits(isSimple) {} static std::string getGraphName(const PGOUseFunc *G) { - return G->getFunc().getName(); + return std::string(G->getFunc().getName()); } std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph) { |