summaryrefslogtreecommitdiff
path: root/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Instrumentation/PGOInstrumentation.cpp')
-rw-r--r--lib/Transforms/Instrumentation/PGOInstrumentation.cpp308
1 files changed, 218 insertions, 90 deletions
diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 8e4bfc0b91bc..cb4b3a9c2545 100644
--- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1,4 +1,4 @@
-//===-- PGOInstrumentation.cpp - MST-based PGO Instrumentation ------------===//
+//===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -50,36 +50,69 @@
#include "llvm/Transforms/PGOInstrumentation.h"
#include "CFGMST.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/IndirectCallSiteVisitor.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
+#include "llvm/IR/Comdat.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfileSummary.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
+#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
-#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/JamCRC.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
@@ -166,15 +199,18 @@ static cl::opt<bool>
cl::desc("Use this option to turn on/off SELECT "
"instruction instrumentation. "));
-// Command line option to turn on CFG dot dump of raw profile counts
-static cl::opt<bool>
- PGOViewRawCounts("pgo-view-raw-counts", cl::init(false), cl::Hidden,
- cl::desc("A boolean option to show CFG dag "
- "with raw profile counts from "
- "profile data. See also option "
- "-pgo-view-counts. To limit graph "
- "display to only one function, use "
- "filtering option -view-bfi-func-name."));
+// Command line option to turn on CFG dot or text dump of raw profile counts
+static cl::opt<PGOViewCountsType> PGOViewRawCounts(
+ "pgo-view-raw-counts", cl::Hidden,
+ cl::desc("A boolean option to show CFG dag or text "
+ "with raw profile counts from "
+ "profile data. See also option "
+ "-pgo-view-counts. To limit graph "
+ "display to only one function, use "
+ "filtering option -view-bfi-func-name."),
+ cl::values(clEnumValN(PGOVCT_None, "none", "do not show."),
+ clEnumValN(PGOVCT_Graph, "graph", "show a graph."),
+ clEnumValN(PGOVCT_Text, "text", "show in text.")));
// Command line option to enable/disable memop intrinsic call.size profiling.
static cl::opt<bool>
@@ -192,17 +228,15 @@ static cl::opt<bool>
// Command line option to turn on CFG dot dump after profile annotation.
// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
-extern cl::opt<bool> PGOViewCounts;
+extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Command line option to specify the name of the function for CFG dump
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;
-namespace {
-
// Return a string describing the branch condition that can be
// used in static branch probability heuristics:
-std::string getBranchCondString(Instruction *TI) {
+static std::string getBranchCondString(Instruction *TI) {
BranchInst *BI = dyn_cast<BranchInst>(TI);
if (!BI || !BI->isConditional())
return std::string();
@@ -233,6 +267,8 @@ std::string getBranchCondString(Instruction *TI) {
return result;
}
+namespace {
+
/// The select instruction visitor plays three roles specified
/// by the mode. In \c VM_counting mode, it simply counts the number of
/// select instructions. In \c VM_instrument mode, it inserts code to count
@@ -259,6 +295,7 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
Mode = VM_counting;
visit(Func);
}
+
// Visit the IR stream and instrument all select instructions. \p
// Ind is a pointer to the counter index variable; \p TotalNC
// is the total number of counters; \p FNV is the pointer to the
@@ -283,8 +320,10 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
void instrumentOneSelectInst(SelectInst &SI);
void annotateOneSelectInst(SelectInst &SI);
+
// Visit \p SI instruction and perform tasks according to visit mode.
void visitSelectInst(SelectInst &SI);
+
// Return the number of select instructions. This needs be called after
// countSelects().
unsigned getNumOfSelectInsts() const { return NSIs; }
@@ -328,8 +367,10 @@ struct MemIntrinsicVisitor : public InstVisitor<MemIntrinsicVisitor> {
// Visit the IR stream and annotate all mem intrinsic call instructions.
void instrumentOneMemIntrinsic(MemIntrinsic &MI);
+
// Visit \p MI instruction and perform tasks according to visit mode.
void visitMemIntrinsic(MemIntrinsic &SI);
+
unsigned getNumOfMemIntrinsics() const { return NMemIs; }
};
@@ -371,6 +412,7 @@ private:
std::string ProfileFileName;
bool runOnModule(Module &M) override;
+
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<BlockFrequencyInfoWrapperPass>();
}
@@ -379,6 +421,7 @@ private:
} // end anonymous namespace
char PGOInstrumentationGenLegacyPass::ID = 0;
+
INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
"PGO instrumentation.", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
@@ -391,6 +434,7 @@ ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
}
char PGOInstrumentationUseLegacyPass::ID = 0;
+
INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
"Read PGO instrumentation profile.", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
@@ -403,6 +447,7 @@ ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
}
namespace {
+
/// \brief An MST based instrumentation for PGO
///
/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
@@ -413,12 +458,13 @@ struct PGOEdge {
const BasicBlock *SrcBB;
const BasicBlock *DestBB;
uint64_t Weight;
- bool InMST;
- bool Removed;
- bool IsCritical;
- PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : SrcBB(Src), DestBB(Dest), Weight(W), InMST(false), Removed(false),
- IsCritical(false) {}
+ bool InMST = false;
+ bool Removed = false;
+ bool IsCritical = false;
+
+ PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
+ : SrcBB(Src), DestBB(Dest), Weight(W) {}
+
// Return the information string of an edge.
const std::string infoString() const {
return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
@@ -430,9 +476,9 @@ struct PGOEdge {
struct BBInfo {
BBInfo *Group;
uint32_t Index;
- uint32_t Rank;
+ uint32_t Rank = 0;
- BBInfo(unsigned IX) : Group(this), Index(IX), Rank(0) {}
+ BBInfo(unsigned IX) : Group(this), Index(IX) {}
// Return the information string of this object.
const std::string infoString() const {
@@ -444,19 +490,22 @@ struct BBInfo {
template <class Edge, class BBInfo> class FuncPGOInstrumentation {
private:
Function &F;
- void computeCFGHash();
- void renameComdatFunction();
+
// A map that stores the Comdat group in function F.
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
+ void computeCFGHash();
+ void renameComdatFunction();
+
public:
std::vector<std::vector<Instruction *>> ValueSites;
SelectInstVisitor SIVisitor;
MemIntrinsicVisitor MIVisitor;
std::string FuncName;
GlobalVariable *FuncNameVar;
+
// CFG hash value for this function.
- uint64_t FunctionHash;
+ uint64_t FunctionHash = 0;
// The Minimum Spanning Tree of function CFG.
CFGMST<Edge, BBInfo> MST;
@@ -483,8 +532,7 @@ public:
bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr)
: F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
- SIVisitor(Func), MIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) {
-
+ SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) {
// This should be done before CFG hash computation.
SIVisitor.countSelects(Func);
MIVisitor.countMemIntrinsics(Func);
@@ -495,7 +543,7 @@ public:
FuncName = getPGOFuncName(F);
computeCFGHash();
- if (ComdatMembers.size())
+ if (!ComdatMembers.empty())
renameComdatFunction();
DEBUG(dumpInfo("after CFGMST"));
@@ -523,6 +571,8 @@ public:
}
};
+} // end anonymous namespace
+
// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
// value of each BB in the CFG. The higher 32 bits record the number of edges.
template <class Edge, class BBInfo>
@@ -545,6 +595,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
(uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
(uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+ DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"
+ << " CRC = " << JC.getCRC()
+ << ", Selects = " << SIVisitor.getNumOfSelectInsts()
+ << ", Edges = " << MST.AllEdges.size()
+ << ", ICSites = " << ValueSites[IPVK_IndirectCallTarget].size()
+ << ", Hash = " << FunctionHash << "\n";);
}
// Check if we can safely rename this Comdat function.
@@ -660,6 +716,9 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
static void instrumentOneFunc(
Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
+ // Split indirectbr critical edges here before computing the MST rather than
+ // later in getInstrBB() to avoid invalidating it.
+ SplitIndirectBrCriticalEdges(F, BPI, BFI);
FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI,
BFI);
unsigned NumCounters = FuncInfo.getNumCounters();
@@ -676,7 +735,7 @@ static void instrumentOneFunc(
"Cannot get the Instrumentation point");
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
- {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
Builder.getInt32(I++)});
}
@@ -700,7 +759,7 @@ static void instrumentOneFunc(
"Cannot get the Instrumentation point");
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
- {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
Builder.getInt64(FuncInfo.FunctionHash),
Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
Builder.getInt32(IPVK_IndirectCallTarget),
@@ -713,12 +772,15 @@ static void instrumentOneFunc(
F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash);
}
+namespace {
+
// This class represents a CFG edge in profile use compilation.
struct PGOUseEdge : public PGOEdge {
- bool CountValid;
- uint64_t CountValue;
- PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : PGOEdge(Src, Dest, W), CountValid(false), CountValue(0) {}
+ bool CountValid = false;
+ uint64_t CountValue = 0;
+
+ PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
+ : PGOEdge(Src, Dest, W) {}
// Set edge count value
void setEdgeCount(uint64_t Value) {
@@ -735,22 +797,21 @@ struct PGOUseEdge : public PGOEdge {
}
};
-typedef SmallVector<PGOUseEdge *, 2> DirectEdges;
+using DirectEdges = SmallVector<PGOUseEdge *, 2>;
// This class stores the auxiliary information for each BB.
struct UseBBInfo : public BBInfo {
- uint64_t CountValue;
+ uint64_t CountValue = 0;
bool CountValid;
- int32_t UnknownCountInEdge;
- int32_t UnknownCountOutEdge;
+ int32_t UnknownCountInEdge = 0;
+ int32_t UnknownCountOutEdge = 0;
DirectEdges InEdges;
DirectEdges OutEdges;
- UseBBInfo(unsigned IX)
- : BBInfo(IX), CountValue(0), CountValid(false), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
+
+ UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {}
+
UseBBInfo(unsigned IX, uint64_t C)
- : BBInfo(IX), CountValue(C), CountValid(true), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
+ : BBInfo(IX), CountValue(C), CountValid(true) {}
// Set the profile count value for this BB.
void setBBInfoCount(uint64_t Value) {
@@ -766,6 +827,8 @@ struct UseBBInfo : public BBInfo {
}
};
+} // end anonymous namespace
+
// Sum up the count values for all the edges.
static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
uint64_t Total = 0;
@@ -777,14 +840,17 @@ static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
return Total;
}
+namespace {
+
class PGOUseFunc {
public:
PGOUseFunc(Function &Func, Module *Modu,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
BranchProbabilityInfo *BPI = nullptr,
- BlockFrequencyInfo *BFI = nullptr)
- : F(Func), M(Modu), FuncInfo(Func, ComdatMembers, false, BPI, BFI),
- CountPosition(0), ProfileCountSize(0), FreqAttr(FFA_Normal) {}
+ BlockFrequencyInfo *BFIin = nullptr)
+ : F(Func), M(Modu), BFI(BFIin),
+ FuncInfo(Func, ComdatMembers, false, BPI, BFIin),
+ FreqAttr(FFA_Normal) {}
// Read counts for the instrumented BB from profile.
bool readCounters(IndexedInstrProfReader *PGOReader);
@@ -801,6 +867,9 @@ public:
// Annotate the value profile call sites for one value kind.
void annotateValueSites(uint32_t Kind);
+ // Annotate the irreducible loop header weights.
+ void annotateIrrLoopHeaderWeights();
+
// The hotness of the function from the profile count.
enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
@@ -809,6 +878,7 @@ public:
// Return the function hash.
uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }
+
// Return the profile record for this function;
InstrProfRecord &getProfileRecord() { return ProfileRecord; }
@@ -824,9 +894,15 @@ public:
Function &getFunc() const { return F; }
+ void dumpInfo(std::string Str = "") const {
+ FuncInfo.dumpInfo(Str);
+ }
+
private:
Function &F;
Module *M;
+ BlockFrequencyInfo *BFI;
+
// This member stores the shared information with class PGOGenFunc.
FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
@@ -835,10 +911,10 @@ private:
uint64_t ProgramMaxCount;
// Position of counter that remains to be read.
- uint32_t CountPosition;
+ uint32_t CountPosition = 0;
// Total size of the profile count for this function.
- uint32_t ProfileCountSize;
+ uint32_t ProfileCountSize = 0;
// ProfileRecord for this function.
InstrProfRecord ProfileRecord;
@@ -873,11 +949,12 @@ private:
}
};
+} // end anonymous namespace
+
// Visit all the edges and assign the count value for the instrumented
// edges and the BB.
void PGOUseFunc::setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile) {
-
assert(FuncInfo.getNumCounters() == CountFromProfile.size());
// Use a worklist as we will update the vector during the iteration.
std::vector<PGOUseEdge *> WorkList;
@@ -1087,7 +1164,8 @@ void PGOUseFunc::setBranchWeights() {
TerminatorInst *TI = BB.getTerminator();
if (TI->getNumSuccessors() < 2)
continue;
- if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
+ if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
+ isa<IndirectBrInst>(TI)))
continue;
if (getBBInfo(&BB).CountValue == 0)
continue;
@@ -1113,6 +1191,29 @@ void PGOUseFunc::setBranchWeights() {
}
}
+static bool isIndirectBrTarget(BasicBlock *BB) {
+ for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
+ if (isa<IndirectBrInst>((*PI)->getTerminator()))
+ return true;
+ }
+ return false;
+}
+
+void PGOUseFunc::annotateIrrLoopHeaderWeights() {
+ DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n");
+ // Find irr loop headers
+ for (auto &BB : F) {
+ // As a heuristic also annotate indrectbr targets as they have a high chance
+ // to become an irreducible loop header after the indirectbr tail
+ // duplication.
+ if (BFI->isIrrLoopHeader(&BB) || isIndirectBrTarget(&BB)) {
+ TerminatorInst *TI = BB.getTerminator();
+ const UseBBInfo &BBCountInfo = getBBInfo(&BB);
+ setIrrLoopHeaderMetadata(M, TI, BBCountInfo.CountValue);
+ }
+ }
+}
+
void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
Module *M = F.getParent();
IRBuilder<> Builder(&SI);
@@ -1121,7 +1222,7 @@ void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step),
- {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
Builder.getInt32(*CurCtrIdx), Step});
++(*CurCtrIdx);
@@ -1176,7 +1277,7 @@ void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) {
assert(!dyn_cast<ConstantInt>(Length));
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
- {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty),
Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)});
++CurCtrId;
@@ -1242,7 +1343,6 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) {
ValueSiteIndex++;
}
}
-} // end anonymous namespace
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
@@ -1312,7 +1412,6 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
PreservedAnalyses PGOInstrumentationGen::run(Module &M,
ModuleAnalysisManager &AM) {
-
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
auto LookupBPI = [&FAM](Function &F) {
return &FAM.getResult<BranchProbabilityAnalysis>(F);
@@ -1367,33 +1466,48 @@ static bool annotateAllFunctions(
continue;
auto *BPI = LookupBPI(F);
auto *BFI = LookupBFI(F);
+ // Split indirectbr critical edges here before computing the MST rather than
+ // later in getInstrBB() to avoid invalidating it.
+ SplitIndirectBrCriticalEdges(F, BPI, BFI);
PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI);
if (!Func.readCounters(PGOReader.get()))
continue;
Func.populateCounters();
Func.setBranchWeights();
Func.annotateValueSites();
+ Func.annotateIrrLoopHeaderWeights();
PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
if (FreqAttr == PGOUseFunc::FFA_Cold)
ColdFunctions.push_back(&F);
else if (FreqAttr == PGOUseFunc::FFA_Hot)
HotFunctions.push_back(&F);
- if (PGOViewCounts && (ViewBlockFreqFuncName.empty() ||
- F.getName().equals(ViewBlockFreqFuncName))) {
+ if (PGOViewCounts != PGOVCT_None &&
+ (ViewBlockFreqFuncName.empty() ||
+ F.getName().equals(ViewBlockFreqFuncName))) {
LoopInfo LI{DominatorTree(F)};
std::unique_ptr<BranchProbabilityInfo> NewBPI =
llvm::make_unique<BranchProbabilityInfo>(F, LI);
std::unique_ptr<BlockFrequencyInfo> NewBFI =
llvm::make_unique<BlockFrequencyInfo>(F, *NewBPI, LI);
-
- NewBFI->view();
+ if (PGOViewCounts == PGOVCT_Graph)
+ NewBFI->view();
+ else if (PGOViewCounts == PGOVCT_Text) {
+ dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n";
+ NewBFI->print(dbgs());
+ }
}
- if (PGOViewRawCounts && (ViewBlockFreqFuncName.empty() ||
- F.getName().equals(ViewBlockFreqFuncName))) {
- if (ViewBlockFreqFuncName.empty())
- WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
- else
- ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
+ if (PGOViewRawCounts != PGOVCT_None &&
+ (ViewBlockFreqFuncName.empty() ||
+ F.getName().equals(ViewBlockFreqFuncName))) {
+ if (PGOViewRawCounts == PGOVCT_Graph)
+ if (ViewBlockFreqFuncName.empty())
+ WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
+ else
+ ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
+ else if (PGOViewRawCounts == PGOVCT_Text) {
+ dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n";
+ Func.dumpInfo();
+ }
}
}
M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
@@ -1402,12 +1516,12 @@ static bool annotateAllFunctions(
// can affect the BranchProbabilityInfo of any callers, resulting in an
// inconsistent MST between prof-gen and prof-use.
for (auto &F : HotFunctions) {
- F->addFnAttr(llvm::Attribute::InlineHint);
+ F->addFnAttr(Attribute::InlineHint);
DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()
<< "\n");
}
for (auto &F : ColdFunctions) {
- F->addFnAttr(llvm::Attribute::Cold);
+ F->addFnAttr(Attribute::Cold);
DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n");
}
return true;
@@ -1451,9 +1565,19 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI);
}
-namespace llvm {
-void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
- uint64_t MaxCount) {
+static std::string getSimpleNodeName(const BasicBlock *Node) {
+ if (!Node->getName().empty())
+ return Node->getName();
+
+ std::string SimpleNodeName;
+ raw_string_ostream OS(SimpleNodeName);
+ Node->printAsOperand(OS, false);
+ return OS.str();
+}
+
+void llvm::setProfMetadata(Module *M, Instruction *TI,
+ ArrayRef<uint64_t> EdgeCounts,
+ uint64_t MaxCount) {
MDBuilder MDB(M->getContext());
assert(MaxCount > 0 && "Bad max count");
uint64_t Scale = calculateCountScale(MaxCount);
@@ -1464,7 +1588,7 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
DEBUG(dbgs() << "Weight is: ";
for (const auto &W : Weights) { dbgs() << W << " "; }
dbgs() << "\n";);
- TI->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+ TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
if (EmitBranchProbability) {
std::string BrCondStr = getBranchCondString(TI);
if (BrCondStr.empty())
@@ -1483,43 +1607,46 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
OS << " (total count : " << TotalCount << ")";
OS.flush();
Function *F = TI->getParent()->getParent();
- emitOptimizationRemarkAnalysis(
- F->getContext(), "pgo-use-annot", *F, TI->getDebugLoc(),
- Twine(BrCondStr) +
- " is true with probability : " + Twine(BranchProbStr));
+ OptimizationRemarkEmitter ORE(F);
+ ORE.emit([&]() {
+ return OptimizationRemark(DEBUG_TYPE, "pgo-instrumentation", TI)
+ << BrCondStr << " is true with probability : " << BranchProbStr;
+ });
}
}
+namespace llvm {
+
+void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) {
+ MDBuilder MDB(M->getContext());
+ TI->setMetadata(llvm::LLVMContext::MD_irr_loop,
+ MDB.createIrrLoopHeaderWeight(Count));
+}
+
template <> struct GraphTraits<PGOUseFunc *> {
- typedef const BasicBlock *NodeRef;
- typedef succ_const_iterator ChildIteratorType;
- typedef pointer_iterator<Function::const_iterator> nodes_iterator;
+ using NodeRef = const BasicBlock *;
+ using ChildIteratorType = succ_const_iterator;
+ using nodes_iterator = pointer_iterator<Function::const_iterator>;
static NodeRef getEntryNode(const PGOUseFunc *G) {
return &G->getFunc().front();
}
+
static ChildIteratorType child_begin(const NodeRef N) {
return succ_begin(N);
}
+
static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
+
static nodes_iterator nodes_begin(const PGOUseFunc *G) {
return nodes_iterator(G->getFunc().begin());
}
+
static nodes_iterator nodes_end(const PGOUseFunc *G) {
return nodes_iterator(G->getFunc().end());
}
};
-static std::string getSimpleNodeName(const BasicBlock *Node) {
- if (!Node->getName().empty())
- return Node->getName();
-
- std::string SimpleNodeName;
- raw_string_ostream OS(SimpleNodeName);
- Node->printAsOperand(OS, false);
- return OS.str();
-}
-
template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
explicit DOTGraphTraits(bool isSimple = false)
: DefaultDOTGraphTraits(isSimple) {}
@@ -1559,4 +1686,5 @@ template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
return Result;
}
};
-} // namespace llvm
+
+} // end namespace llvm
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-08 16:08:02 +0000