diff options
Diffstat (limited to 'lib/Transforms/Instrumentation/PGOInstrumentation.cpp')
| -rw-r--r-- | lib/Transforms/Instrumentation/PGOInstrumentation.cpp | 409 |
1 files changed, 322 insertions, 87 deletions
diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 4b59b93b325f4..f54d8ad481462 100644 --- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -25,9 +25,12 @@ // // This file contains two passes: // (1) Pass PGOInstrumentationGen which instruments the IR to generate edge -// count profile, and +// count profile, and generates the instrumentation for indirect call +// profiling. // (2) Pass PGOInstrumentationUse which reads the edge count profile and -// annotates the branch weights. +// annotates the branch weights. It also reads the indirect call value +// profiling records and annotate the indirect call instructions. +// // To get the precise counter information, These two passes need to invoke at // the same compilation point (so they see the same IR). For pass // PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For @@ -45,14 +48,16 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/PGOInstrumentation.h" #include "CFGMST.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Triple.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/IndirectCallSiteVisitor.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" @@ -62,10 +67,13 @@ #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Debug.h" #include "llvm/Support/JamCRC.h" +#include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include <algorithm> #include <string> #include <utility> #include <vector> @@ -81,6 +89,7 @@ STATISTIC(NumOfPGOSplit, "Number of critical edge splits."); STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts."); STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); +STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); // Command line option to specify the file to read profile from. This is // mainly used for testing. @@ -90,13 +99,37 @@ static cl::opt<std::string> cl::desc("Specify the path of profile data file. This is" "mainly for test purpose.")); +// Command line option to disable value profiling. The default is false: +// i.e. value profiling is enabled by default. This is for debug purpose. +static cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false), + cl::Hidden, + cl::desc("Disable Value Profiling")); + +// Command line option to set the maximum number of VP annotations to write to +// the metadata for a single indirect call callsite. +static cl::opt<unsigned> MaxNumAnnotations( + "icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, + cl::desc("Max number of annotations for a single indirect " + "call callsite")); + +// Command line option to enable/disable the warning about missing profile +// information. +static cl::opt<bool> NoPGOWarnMissing("no-pgo-warn-missing", cl::init(false), + cl::Hidden); + +// Command line option to enable/disable the warning about a hash mismatch in +// the profile data. +static cl::opt<bool> NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), + cl::Hidden); + namespace { -class PGOInstrumentationGen : public ModulePass { +class PGOInstrumentationGenLegacyPass : public ModulePass { public: static char ID; - PGOInstrumentationGen() : ModulePass(ID) { - initializePGOInstrumentationGenPass(*PassRegistry::getPassRegistry()); + PGOInstrumentationGenLegacyPass() : ModulePass(ID) { + initializePGOInstrumentationGenLegacyPassPass( + *PassRegistry::getPassRegistry()); } const char *getPassName() const override { @@ -111,16 +144,17 @@ private: } }; -class PGOInstrumentationUse : public ModulePass { +class PGOInstrumentationUseLegacyPass : public ModulePass { public: static char ID; // Provide the profile filename as the parameter. - PGOInstrumentationUse(std::string Filename = "") - : ModulePass(ID), ProfileFileName(Filename) { + PGOInstrumentationUseLegacyPass(std::string Filename = "") + : ModulePass(ID), ProfileFileName(std::move(Filename)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; - initializePGOInstrumentationUsePass(*PassRegistry::getPassRegistry()); + initializePGOInstrumentationUseLegacyPassPass( + *PassRegistry::getPassRegistry()); } const char *getPassName() const override { @@ -129,37 +163,36 @@ public: private: std::string ProfileFileName; - std::unique_ptr<IndexedInstrProfReader> PGOReader; - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<BlockFrequencyInfoWrapperPass>(); } }; } // end anonymous namespace -char PGOInstrumentationGen::ID = 0; -INITIALIZE_PASS_BEGIN(PGOInstrumentationGen, "pgo-instr-gen", +char PGOInstrumentationGenLegacyPass::ID = 0; +INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) -INITIALIZE_PASS_END(PGOInstrumentationGen, "pgo-instr-gen", +INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) -ModulePass *llvm::createPGOInstrumentationGenPass() { - return new PGOInstrumentationGen(); +ModulePass *llvm::createPGOInstrumentationGenLegacyPass() { + return new PGOInstrumentationGenLegacyPass(); } -char PGOInstrumentationUse::ID = 0; -INITIALIZE_PASS_BEGIN(PGOInstrumentationUse, "pgo-instr-use", +char PGOInstrumentationUseLegacyPass::ID = 0; +INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) -INITIALIZE_PASS_END(PGOInstrumentationUse, "pgo-instr-use", +INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) -ModulePass *llvm::createPGOInstrumentationUsePass(StringRef Filename) { - return new PGOInstrumentationUse(Filename.str()); +ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) { + return new PGOInstrumentationUseLegacyPass(Filename.str()); } namespace { @@ -225,7 +258,7 @@ public: // Dump edges and BB information. void dumpInfo(std::string Str = "") const { MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " + - Twine(FunctionHash) + "\t" + Str); + Twine(FunctionHash) + "\t" + Str); } FuncPGOInstrumentation(Function &Func, bool CreateGlobalVar = false, @@ -247,7 +280,7 @@ public: if (CreateGlobalVar) FuncNameVar = createPGOFuncNameVar(F, FuncName); - }; + } }; // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index @@ -305,7 +338,7 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) { return InstrBB; } -// Visit all edge and instrument the edges not in MST. +// Visit all edge and instrument the edges not in MST, and do value profiling. // Critical edges will be split. static void instrumentOneFunc(Function &F, Module *M, BranchProbabilityInfo *BPI, @@ -318,6 +351,7 @@ static void instrumentOneFunc(Function &F, Module *M, } uint32_t I = 0; + Type *I8PtrTy = Type::getInt8PtrTy(M->getContext()); for (auto &E : FuncInfo.MST.AllEdges) { BasicBlock *InstrBB = FuncInfo.getInstrBB(E.get()); if (!InstrBB) @@ -326,13 +360,34 @@ static void instrumentOneFunc(Function &F, Module *M, IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt()); assert(Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"); - Type *I8PtrTy = Type::getInt8PtrTy(M->getContext()); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment), {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters), Builder.getInt32(I++)}); } + + if (DisableValueProfiling) + return; + + unsigned NumIndirectCallSites = 0; + for (auto &I : findIndirectCallSites(F)) { + CallSite CS(I); + Value *Callee = CS.getCalledValue(); + DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = " + << NumIndirectCallSites << "\n"); + IRBuilder<> Builder(I); + assert(Builder.GetInsertPoint() != I->getParent()->end() && + "Cannot get the Instrumentation point"); + Builder.CreateCall( + Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), + {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), + Builder.getInt64(FuncInfo.FunctionHash), + Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()), + Builder.getInt32(llvm::InstrProfValueKind::IPVK_IndirectCallTarget), + Builder.getInt32(NumIndirectCallSites++)}); + } + NumOfPGOICall += NumIndirectCallSites; } // This class represents a CFG edge in profile use compilation. @@ -352,7 +407,8 @@ struct PGOUseEdge : public PGOEdge { const std::string infoString() const { if (!CountValid) return PGOEdge::infoString(); - return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue)).str(); + return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue)) + .str(); } }; @@ -399,6 +455,33 @@ static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) { } class PGOUseFunc { +public: + PGOUseFunc(Function &Func, Module *Modu, BranchProbabilityInfo *BPI = nullptr, + BlockFrequencyInfo *BFI = nullptr) + : F(Func), M(Modu), FuncInfo(Func, false, BPI, BFI), + FreqAttr(FFA_Normal) {} + + // Read counts for the instrumented BB from profile. + bool readCounters(IndexedInstrProfReader *PGOReader); + + // Populate the counts for all BBs. + void populateCounters(); + + // Set the branch weights based on the count values. + void setBranchWeights(); + + // Annotate the indirect call sites. + void annotateIndirectCallSites(); + + // The hotness of the function from the profile count. + enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot }; + + // Return the function hotness from the profile. + FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; } + + // Return the profile record for this function; + InstrProfRecord &getProfileRecord() { return ProfileRecord; } + private: Function &F; Module *M; @@ -414,6 +497,12 @@ private: // compilation. uint64_t ProgramMaxCount; + // ProfileRecord for this function. + InstrProfRecord ProfileRecord; + + // Function hotness info derived from profile. + FuncFreqAttr FreqAttr; + // Find the Instrumented BB and set the value. void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile); @@ -427,7 +516,7 @@ private: // Set the hot/cold inline hints based on the count values. // FIXME: This function should be removed once the functionality in // the inliner is implemented. - void applyFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) { + void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) { if (ProgramMaxCount == 0) return; // Threshold of the hot functions. @@ -435,24 +524,10 @@ private: // Threshold of the cold functions. const BranchProbability ColdFunctionThreshold(2, 10000); if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount)) - F.addFnAttr(llvm::Attribute::InlineHint); + FreqAttr = FFA_Hot; else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount)) - F.addFnAttr(llvm::Attribute::Cold); + FreqAttr = FFA_Cold; } - -public: - PGOUseFunc(Function &Func, Module *Modu, BranchProbabilityInfo *BPI = nullptr, - BlockFrequencyInfo *BFI = nullptr) - : F(Func), M(Modu), FuncInfo(Func, false, BPI, BFI) {} - - // Read counts for the instrumented BB from profile. - bool readCounters(IndexedInstrProfReader *PGOReader); - - // Populate the counts for all BBs. - void populateCounters(); - - // Set the branch weights based on the count values. - void setBranchWeights(); }; // Visit all the edges and assign the count value for the instrumented @@ -511,21 +586,32 @@ void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) { // Return true if the profile are successfully read, and false on errors. bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) { auto &Ctx = M->getContext(); - ErrorOr<InstrProfRecord> Result = + Expected<InstrProfRecord> Result = PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash); - if (std::error_code EC = Result.getError()) { - if (EC == instrprof_error::unknown_function) - NumOfPGOMissing++; - else if (EC == instrprof_error::hash_mismatch || - EC == llvm::instrprof_error::malformed) - NumOfPGOMismatch++; - - std::string Msg = EC.message() + std::string(" ") + F.getName().str(); - Ctx.diagnose( - DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); + if (Error E = Result.takeError()) { + handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { + auto Err = IPE.get(); + bool SkipWarning = false; + if (Err == instrprof_error::unknown_function) { + NumOfPGOMissing++; + SkipWarning = NoPGOWarnMissing; + } else if (Err == instrprof_error::hash_mismatch || + Err == instrprof_error::malformed) { + NumOfPGOMismatch++; + SkipWarning = NoPGOWarnMismatch; + } + + if (SkipWarning) + return; + + std::string Msg = IPE.message() + std::string(" ") + F.getName().str(); + Ctx.diagnose( + DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); + }); return false; } - std::vector<uint64_t> &CountFromProfile = Result.get().Counts; + ProfileRecord = std::move(Result.get()); + std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts; NumOfPGOFunc++; DEBUG(dbgs() << CountFromProfile.size() << " counts\n"); @@ -605,16 +691,17 @@ void PGOUseFunc::populateCounters() { } DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n"); +#ifndef NDEBUG // Assert every BB has a valid counter. + for (auto &BB : F) + assert(getBBInfo(&BB).CountValid && "BB count is not valid"); +#endif uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue; + F.setEntryCount(FuncEntryCount); uint64_t FuncMaxCount = FuncEntryCount; - for (auto &BB : F) { - assert(getBBInfo(&BB).CountValid && "BB count is not valid"); - uint64_t Count = getBBInfo(&BB).CountValue; - if (Count > FuncMaxCount) - FuncMaxCount = Count; - } - applyFunctionAttributes(FuncEntryCount, FuncMaxCount); + for (auto &BB : F) + FuncMaxCount = std::max(FuncMaxCount, getBBInfo(&BB).CountValue); + markFunctionAttributes(FuncEntryCount, FuncMaxCount); DEBUG(FuncInfo.dumpInfo("after reading profile.")); } @@ -642,7 +729,7 @@ void PGOUseFunc::setBranchWeights() { const PGOUseEdge *E = BBCountInfo.OutEdges[s]; const BasicBlock *SrcBB = E->SrcBB; const BasicBlock *DestBB = E->DestBB; - if (DestBB == 0) + if (DestBB == nullptr) continue; unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); uint64_t EdgeCount = E->CountValue; @@ -663,56 +750,204 @@ void PGOUseFunc::setBranchWeights() { dbgs() << "\n";); } } + +// Traverse all the indirect callsites and annotate the instructions. +void PGOUseFunc::annotateIndirectCallSites() { + if (DisableValueProfiling) + return; + + // Create the PGOFuncName meta data. + createPGOFuncNameMetadata(F, FuncInfo.FuncName); + + unsigned IndirectCallSiteIndex = 0; + auto IndirectCallSites = findIndirectCallSites(F); + unsigned NumValueSites = + ProfileRecord.getNumValueSites(IPVK_IndirectCallTarget); + if (NumValueSites != IndirectCallSites.size()) { + std::string Msg = + std::string("Inconsistent number of indirect call sites: ") + + F.getName().str(); + auto &Ctx = M->getContext(); + Ctx.diagnose( + DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); + return; + } + + for (auto &I : IndirectCallSites) { + DEBUG(dbgs() << "Read one indirect call instrumentation: Index=" + << IndirectCallSiteIndex << " out of " << NumValueSites + << "\n"); + annotateValueSite(*M, *I, ProfileRecord, IPVK_IndirectCallTarget, + IndirectCallSiteIndex, MaxNumAnnotations); + IndirectCallSiteIndex++; + } +} } // end anonymous namespace -bool PGOInstrumentationGen::runOnModule(Module &M) { +// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime +// aware this is an ir_level profile so it can set the version flag. +static void createIRLevelProfileFlagVariable(Module &M) { + Type *IntTy64 = Type::getInt64Ty(M.getContext()); + uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF); + auto IRLevelVersionVariable = new GlobalVariable( + M, IntTy64, true, GlobalVariable::ExternalLinkage, + Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), + INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR)); + IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); + Triple TT(M.getTargetTriple()); + if (!TT.supportsCOMDAT()) + IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage); + else + IRLevelVersionVariable->setComdat(M.getOrInsertComdat( + StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR)))); +} + +static bool InstrumentAllFunctions( + Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, + function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { + createIRLevelProfileFlagVariable(M); for (auto &F : M) { if (F.isDeclaration()) continue; - BranchProbabilityInfo *BPI = - &(getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI()); - BlockFrequencyInfo *BFI = - &(getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI()); + auto *BPI = LookupBPI(F); + auto *BFI = LookupBFI(F); instrumentOneFunc(F, &M, BPI, BFI); } return true; } -static void setPGOCountOnFunc(PGOUseFunc &Func, - IndexedInstrProfReader *PGOReader) { - if (Func.readCounters(PGOReader)) { - Func.populateCounters(); - Func.setBranchWeights(); - } +bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { + if (skipModule(M)) + return false; + + auto LookupBPI = [this](Function &F) { + return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI(); + }; + auto LookupBFI = [this](Function &F) { + return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); + }; + return InstrumentAllFunctions(M, LookupBPI, LookupBFI); } -bool PGOInstrumentationUse::runOnModule(Module &M) { +PreservedAnalyses PGOInstrumentationGen::run(Module &M, + AnalysisManager<Module> &AM) { + + auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + auto LookupBPI = [&FAM](Function &F) { + return &FAM.getResult<BranchProbabilityAnalysis>(F); + }; + + auto LookupBFI = [&FAM](Function &F) { + return &FAM.getResult<BlockFrequencyAnalysis>(F); + }; + + if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI)) + return PreservedAnalyses::all(); + + return PreservedAnalyses::none(); +} + +static bool annotateAllFunctions( + Module &M, StringRef ProfileFileName, + function_ref<BranchProbabilityInfo *(Function &)> LookupBPI, + function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName); - if (std::error_code EC = ReaderOrErr.getError()) { - Ctx.diagnose( - DiagnosticInfoPGOProfile(ProfileFileName.data(), EC.message())); + if (Error E = ReaderOrErr.takeError()) { + handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { + Ctx.diagnose( + DiagnosticInfoPGOProfile(ProfileFileName.data(), EI.message())); + }); return false; } - PGOReader = std::move(ReaderOrErr.get()); + std::unique_ptr<IndexedInstrProfReader> PGOReader = + std::move(ReaderOrErr.get()); if (!PGOReader) { Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(), - "Cannot get PGOReader")); + StringRef("Cannot get PGOReader"))); + return false; + } + // TODO: might need to change the warning once the clang option is finalized. + if (!PGOReader->isIRLevelProfile()) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + ProfileFileName.data(), "Not an IR level instrumentation profile")); return false; } + std::vector<Function *> HotFunctions; + std::vector<Function *> ColdFunctions; for (auto &F : M) { if (F.isDeclaration()) continue; - BranchProbabilityInfo *BPI = - &(getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI()); - BlockFrequencyInfo *BFI = - &(getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI()); + auto *BPI = LookupBPI(F); + auto *BFI = LookupBFI(F); PGOUseFunc Func(F, &M, BPI, BFI); - setPGOCountOnFunc(Func, PGOReader.get()); + if (!Func.readCounters(PGOReader.get())) + continue; + Func.populateCounters(); + Func.setBranchWeights(); + Func.annotateIndirectCallSites(); + PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr(); + if (FreqAttr == PGOUseFunc::FFA_Cold) + ColdFunctions.push_back(&F); + else if (FreqAttr == PGOUseFunc::FFA_Hot) + HotFunctions.push_back(&F); + } + M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext())); + // Set function hotness attribute from the profile. + // We have to apply these attributes at the end because their presence + // can affect the BranchProbabilityInfo of any callers, resulting in an + // inconsistent MST between prof-gen and prof-use. + for (auto &F : HotFunctions) { + F->addFnAttr(llvm::Attribute::InlineHint); + DEBUG(dbgs() << "Set inline attribute to function: " << F->getName() + << "\n"); + } + for (auto &F : ColdFunctions) { + F->addFnAttr(llvm::Attribute::Cold); + DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n"); } + return true; } + +PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename) + : ProfileFileName(std::move(Filename)) { + if (!PGOTestProfileFile.empty()) + ProfileFileName = PGOTestProfileFile; +} + +PreservedAnalyses PGOInstrumentationUse::run(Module &M, + AnalysisManager<Module> &AM) { + + auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + auto LookupBPI = [&FAM](Function &F) { + return &FAM.getResult<BranchProbabilityAnalysis>(F); + }; + + auto LookupBFI = [&FAM](Function &F) { + return &FAM.getResult<BlockFrequencyAnalysis>(F); + }; + + if (!annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI)) + return PreservedAnalyses::all(); + + return PreservedAnalyses::none(); +} + +bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { + if (skipModule(M)) + return false; + + auto LookupBPI = [this](Function &F) { + return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI(); + }; + auto LookupBFI = [this](Function &F) { + return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); + }; + + return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI); +} |