summaryrefslogtreecommitdiff
path: root/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Instrumentation/IndirectCallPromotion.cpp')
-rw-r--r--lib/Transforms/Instrumentation/IndirectCallPromotion.cpp472
1 files changed, 116 insertions, 356 deletions
diff --git a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 4089d81ea3e1..49b8a67a6c14 100644
--- a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -1,4 +1,4 @@
-//===-- IndirectCallPromotion.cpp - Optimizations based on value profiling ===//
+//===- IndirectCallPromotion.cpp - Optimizations based on value profiling -===//
//
// The LLVM Compiler Infrastructure
//
@@ -14,13 +14,15 @@
//===----------------------------------------------------------------------===//
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
-#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
#include "llvm/Analysis/IndirectCallSiteVisitor.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/DerivedTypes.h"
@@ -34,20 +36,23 @@
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
-#include "llvm/PassRegistry.h"
-#include "llvm/PassSupport.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/PGOInstrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include <cassert>
#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
#include <vector>
using namespace llvm;
@@ -110,6 +115,7 @@ static cl::opt<bool>
cl::desc("Dump IR after transformation happens"));
namespace {
+
class PGOIndirectCallPromotionLegacyPass : public ModulePass {
public:
static char ID;
@@ -120,6 +126,10 @@ public:
*PassRegistry::getPassRegistry());
}
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<ProfileSummaryInfoWrapperPass>();
+ }
+
StringRef getPassName() const override { return "PGOIndirectCallPromotion"; }
private:
@@ -133,13 +143,20 @@ private:
// the promoted direct call.
bool SamplePGO;
};
+
} // end anonymous namespace
char PGOIndirectCallPromotionLegacyPass::ID = 0;
-INITIALIZE_PASS(PGOIndirectCallPromotionLegacyPass, "pgo-icall-prom",
- "Use PGO instrumentation profile to promote indirect calls to "
- "direct calls.",
- false, false)
+
+INITIALIZE_PASS_BEGIN(PGOIndirectCallPromotionLegacyPass, "pgo-icall-prom",
+ "Use PGO instrumentation profile to promote indirect "
+ "calls to direct calls.",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
+INITIALIZE_PASS_END(PGOIndirectCallPromotionLegacyPass, "pgo-icall-prom",
+ "Use PGO instrumentation profile to promote indirect "
+ "calls to direct calls.",
+ false, false)
ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO,
bool SamplePGO) {
@@ -147,6 +164,7 @@ ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO,
}
namespace {
+
// The class for main data structure to promote indirect calls to conditional
// direct calls.
class ICallPromotionFunc {
@@ -160,14 +178,13 @@ private:
bool SamplePGO;
- // Test if we can legally promote this direct-call of Target.
- bool isPromotionLegal(Instruction *Inst, uint64_t Target, Function *&F,
- const char **Reason = nullptr);
+ OptimizationRemarkEmitter &ORE;
// A struct that records the direct target and it's call count.
struct PromotionCandidate {
Function *TargetFunction;
uint64_t Count;
+
PromotionCandidate(Function *F, uint64_t C) : TargetFunction(F), Count(C) {}
};
@@ -186,72 +203,17 @@ private:
const std::vector<PromotionCandidate> &Candidates,
uint64_t &TotalCount);
- // Noncopyable
- ICallPromotionFunc(const ICallPromotionFunc &other) = delete;
- ICallPromotionFunc &operator=(const ICallPromotionFunc &other) = delete;
-
public:
ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab,
- bool SamplePGO)
- : F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO) {}
+ bool SamplePGO, OptimizationRemarkEmitter &ORE)
+ : F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {}
+ ICallPromotionFunc(const ICallPromotionFunc &) = delete;
+ ICallPromotionFunc &operator=(const ICallPromotionFunc &) = delete;
- bool processFunction();
+ bool processFunction(ProfileSummaryInfo *PSI);
};
-} // end anonymous namespace
-bool llvm::isLegalToPromote(Instruction *Inst, Function *F,
- const char **Reason) {
- // Check the return type.
- Type *CallRetType = Inst->getType();
- if (!CallRetType->isVoidTy()) {
- Type *FuncRetType = F->getReturnType();
- if (FuncRetType != CallRetType &&
- !CastInst::isBitCastable(FuncRetType, CallRetType)) {
- if (Reason)
- *Reason = "Return type mismatch";
- return false;
- }
- }
-
- // Check if the arguments are compatible with the parameters
- FunctionType *DirectCalleeType = F->getFunctionType();
- unsigned ParamNum = DirectCalleeType->getFunctionNumParams();
- CallSite CS(Inst);
- unsigned ArgNum = CS.arg_size();
-
- if (ParamNum != ArgNum && !DirectCalleeType->isVarArg()) {
- if (Reason)
- *Reason = "The number of arguments mismatch";
- return false;
- }
-
- for (unsigned I = 0; I < ParamNum; ++I) {
- Type *PTy = DirectCalleeType->getFunctionParamType(I);
- Type *ATy = CS.getArgument(I)->getType();
- if (PTy == ATy)
- continue;
- if (!CastInst::castIsValid(Instruction::BitCast, CS.getArgument(I), PTy)) {
- if (Reason)
- *Reason = "Argument type mismatch";
- return false;
- }
- }
-
- DEBUG(dbgs() << " #" << NumOfPGOICallPromotion << " Promote the icall to "
- << F->getName() << "\n");
- return true;
-}
-
-bool ICallPromotionFunc::isPromotionLegal(Instruction *Inst, uint64_t Target,
- Function *&TargetFunction,
- const char **Reason) {
- TargetFunction = Symtab->getFunction(Target);
- if (TargetFunction == nullptr) {
- *Reason = "Cannot find the target";
- return false;
- }
- return isLegalToPromote(Inst, TargetFunction, Reason);
-}
+} // end anonymous namespace
// Indirect-call promotion heuristic. The direct targets are sorted based on
// the count. Stop at the first target that is not promoted.
@@ -279,51 +241,63 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
if (ICPInvokeOnly && dyn_cast<CallInst>(Inst)) {
DEBUG(dbgs() << " Not promote: User options.\n");
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst)
+ << " Not promote: User options";
+ });
break;
}
if (ICPCallOnly && dyn_cast<InvokeInst>(Inst)) {
DEBUG(dbgs() << " Not promote: User option.\n");
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", Inst)
+ << " Not promote: User options";
+ });
break;
}
if (ICPCutOff != 0 && NumOfPGOICallPromotion >= ICPCutOff) {
DEBUG(dbgs() << " Not promote: Cutoff reached.\n");
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "CutOffReached", Inst)
+ << " Not promote: Cutoff reached";
+ });
+ break;
+ }
+
+ Function *TargetFunction = Symtab->getFunction(Target);
+ if (TargetFunction == nullptr) {
+ DEBUG(dbgs() << " Not promote: Cannot find the target\n");
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", Inst)
+ << "Cannot promote indirect call: target not found";
+ });
break;
}
- Function *TargetFunction = nullptr;
+
const char *Reason = nullptr;
- if (!isPromotionLegal(Inst, Target, TargetFunction, &Reason)) {
- StringRef TargetFuncName = Symtab->getFuncName(Target);
- DEBUG(dbgs() << " Not promote: " << Reason << "\n");
- emitOptimizationRemarkMissed(
- F.getContext(), "pgo-icall-prom", F, Inst->getDebugLoc(),
- Twine("Cannot promote indirect call to ") +
- (TargetFuncName.empty() ? Twine(Target) : Twine(TargetFuncName)) +
- Twine(" with count of ") + Twine(Count) + ": " + Reason);
+ if (!isLegalToPromote(CallSite(Inst), TargetFunction, &Reason)) {
+ using namespace ore;
+
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", Inst)
+ << "Cannot promote indirect call to "
+ << NV("TargetFunction", TargetFunction) << " with count of "
+ << NV("Count", Count) << ": " << Reason;
+ });
break;
}
+
Ret.push_back(PromotionCandidate(TargetFunction, Count));
TotalCount -= Count;
}
return Ret;
}
-// Create a diamond structure for If_Then_Else. Also update the profile
-// count. Do the fix-up for the invoke instruction.
-static void createIfThenElse(Instruction *Inst, Function *DirectCallee,
- uint64_t Count, uint64_t TotalCount,
- BasicBlock **DirectCallBB,
- BasicBlock **IndirectCallBB,
- BasicBlock **MergeBB) {
- CallSite CS(Inst);
- Value *OrigCallee = CS.getCalledValue();
-
- IRBuilder<> BBBuilder(Inst);
- LLVMContext &Ctx = Inst->getContext();
- Value *BCI1 =
- BBBuilder.CreateBitCast(OrigCallee, Type::getInt8PtrTy(Ctx), "");
- Value *BCI2 =
- BBBuilder.CreateBitCast(DirectCallee, Type::getInt8PtrTy(Ctx), "");
- Value *PtrCmp = BBBuilder.CreateICmpEQ(BCI1, BCI2, "");
+Instruction *llvm::pgo::promoteIndirectCall(Instruction *Inst,
+ Function *DirectCallee,
+ uint64_t Count, uint64_t TotalCount,
+ bool AttachProfToDirectCall,
+ OptimizationRemarkEmitter *ORE) {
uint64_t ElseCount = TotalCount - Count;
uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount);
@@ -331,261 +305,26 @@ static void createIfThenElse(Instruction *Inst, Function *DirectCallee,
MDBuilder MDB(Inst->getContext());
MDNode *BranchWeights = MDB.createBranchWeights(
scaleBranchCount(Count, Scale), scaleBranchCount(ElseCount, Scale));
- TerminatorInst *ThenTerm, *ElseTerm;
- SplitBlockAndInsertIfThenElse(PtrCmp, Inst, &ThenTerm, &ElseTerm,
- BranchWeights);
- *DirectCallBB = ThenTerm->getParent();
- (*DirectCallBB)->setName("if.true.direct_targ");
- *IndirectCallBB = ElseTerm->getParent();
- (*IndirectCallBB)->setName("if.false.orig_indirect");
- *MergeBB = Inst->getParent();
- (*MergeBB)->setName("if.end.icp");
-
- // Special handing of Invoke instructions.
- InvokeInst *II = dyn_cast<InvokeInst>(Inst);
- if (!II)
- return;
-
- // We don't need branch instructions for invoke.
- ThenTerm->eraseFromParent();
- ElseTerm->eraseFromParent();
-
- // Add jump from Merge BB to the NormalDest. This is needed for the newly
- // created direct invoke stmt -- as its NormalDst will be fixed up to MergeBB.
- BranchInst::Create(II->getNormalDest(), *MergeBB);
-}
-
-// Find the PHI in BB that have the CallResult as the operand.
-static bool getCallRetPHINode(BasicBlock *BB, Instruction *Inst) {
- BasicBlock *From = Inst->getParent();
- for (auto &I : *BB) {
- PHINode *PHI = dyn_cast<PHINode>(&I);
- if (!PHI)
- continue;
- int IX = PHI->getBasicBlockIndex(From);
- if (IX == -1)
- continue;
- Value *V = PHI->getIncomingValue(IX);
- if (dyn_cast<Instruction>(V) == Inst)
- return true;
- }
- return false;
-}
-
-// This method fixes up PHI nodes in BB where BB is the UnwindDest of an
-// invoke instruction. In BB, there may be PHIs with incoming block being
-// OrigBB (the MergeBB after if-then-else splitting). After moving the invoke
-// instructions to its own BB, OrigBB is no longer the predecessor block of BB.
-// Instead two new predecessors are added: IndirectCallBB and DirectCallBB,
-// so the PHI node's incoming BBs need to be fixed up accordingly.
-static void fixupPHINodeForUnwind(Instruction *Inst, BasicBlock *BB,
- BasicBlock *OrigBB,
- BasicBlock *IndirectCallBB,
- BasicBlock *DirectCallBB) {
- for (auto &I : *BB) {
- PHINode *PHI = dyn_cast<PHINode>(&I);
- if (!PHI)
- continue;
- int IX = PHI->getBasicBlockIndex(OrigBB);
- if (IX == -1)
- continue;
- Value *V = PHI->getIncomingValue(IX);
- PHI->addIncoming(V, IndirectCallBB);
- PHI->setIncomingBlock(IX, DirectCallBB);
- }
-}
-
-// This method fixes up PHI nodes in BB where BB is the NormalDest of an
-// invoke instruction. In BB, there may be PHIs with incoming block being
-// OrigBB (the MergeBB after if-then-else splitting). After moving the invoke
-// instructions to its own BB, a new incoming edge will be added to the original
-// NormalDstBB from the IndirectCallBB.
-static void fixupPHINodeForNormalDest(Instruction *Inst, BasicBlock *BB,
- BasicBlock *OrigBB,
- BasicBlock *IndirectCallBB,
- Instruction *NewInst) {
- for (auto &I : *BB) {
- PHINode *PHI = dyn_cast<PHINode>(&I);
- if (!PHI)
- continue;
- int IX = PHI->getBasicBlockIndex(OrigBB);
- if (IX == -1)
- continue;
- Value *V = PHI->getIncomingValue(IX);
- if (dyn_cast<Instruction>(V) == Inst) {
- PHI->setIncomingBlock(IX, IndirectCallBB);
- PHI->addIncoming(NewInst, OrigBB);
- continue;
- }
- PHI->addIncoming(V, IndirectCallBB);
- }
-}
-
-// Add a bitcast instruction to the direct-call return value if needed.
-static Instruction *insertCallRetCast(const Instruction *Inst,
- Instruction *DirectCallInst,
- Function *DirectCallee) {
- if (Inst->getType()->isVoidTy())
- return DirectCallInst;
-
- Type *CallRetType = Inst->getType();
- Type *FuncRetType = DirectCallee->getReturnType();
- if (FuncRetType == CallRetType)
- return DirectCallInst;
-
- BasicBlock *InsertionBB;
- if (CallInst *CI = dyn_cast<CallInst>(DirectCallInst))
- InsertionBB = CI->getParent();
- else
- InsertionBB = (dyn_cast<InvokeInst>(DirectCallInst))->getNormalDest();
-
- return (new BitCastInst(DirectCallInst, CallRetType, "",
- InsertionBB->getTerminator()));
-}
-
-// Create a DirectCall instruction in the DirectCallBB.
-// Parameter Inst is the indirect-call (invoke) instruction.
-// DirectCallee is the decl of the direct-call (invoke) target.
-// DirecallBB is the BB that the direct-call (invoke) instruction is inserted.
-// MergeBB is the bottom BB of the if-then-else-diamond after the
-// transformation. For invoke instruction, the edges from DirectCallBB and
-// IndirectCallBB to MergeBB are removed before this call (during
-// createIfThenElse).
-static Instruction *createDirectCallInst(const Instruction *Inst,
- Function *DirectCallee,
- BasicBlock *DirectCallBB,
- BasicBlock *MergeBB) {
- Instruction *NewInst = Inst->clone();
- if (CallInst *CI = dyn_cast<CallInst>(NewInst)) {
- CI->setCalledFunction(DirectCallee);
- CI->mutateFunctionType(DirectCallee->getFunctionType());
- } else {
- // Must be an invoke instruction. Direct invoke's normal destination is
- // fixed up to MergeBB. MergeBB is the place where return cast is inserted.
- // Also since IndirectCallBB does not have an edge to MergeBB, there is no
- // need to insert new PHIs into MergeBB.
- InvokeInst *II = dyn_cast<InvokeInst>(NewInst);
- assert(II);
- II->setCalledFunction(DirectCallee);
- II->mutateFunctionType(DirectCallee->getFunctionType());
- II->setNormalDest(MergeBB);
- }
-
- DirectCallBB->getInstList().insert(DirectCallBB->getFirstInsertionPt(),
- NewInst);
-
- // Clear the value profile data.
- NewInst->setMetadata(LLVMContext::MD_prof, nullptr);
- CallSite NewCS(NewInst);
- FunctionType *DirectCalleeType = DirectCallee->getFunctionType();
- unsigned ParamNum = DirectCalleeType->getFunctionNumParams();
- for (unsigned I = 0; I < ParamNum; ++I) {
- Type *ATy = NewCS.getArgument(I)->getType();
- Type *PTy = DirectCalleeType->getParamType(I);
- if (ATy != PTy) {
- BitCastInst *BI = new BitCastInst(NewCS.getArgument(I), PTy, "", NewInst);
- NewCS.setArgument(I, BI);
- }
- }
-
- return insertCallRetCast(Inst, NewInst, DirectCallee);
-}
-
-// Create a PHI to unify the return values of calls.
-static void insertCallRetPHI(Instruction *Inst, Instruction *CallResult,
- Function *DirectCallee) {
- if (Inst->getType()->isVoidTy())
- return;
-
- BasicBlock *RetValBB = CallResult->getParent();
-
- BasicBlock *PHIBB;
- if (InvokeInst *II = dyn_cast<InvokeInst>(CallResult))
- RetValBB = II->getNormalDest();
-
- PHIBB = RetValBB->getSingleSuccessor();
- if (getCallRetPHINode(PHIBB, Inst))
- return;
-
- PHINode *CallRetPHI = PHINode::Create(Inst->getType(), 0);
- PHIBB->getInstList().push_front(CallRetPHI);
- Inst->replaceAllUsesWith(CallRetPHI);
- CallRetPHI->addIncoming(Inst, Inst->getParent());
- CallRetPHI->addIncoming(CallResult, RetValBB);
-}
-
-// This function does the actual indirect-call promotion transformation:
-// For an indirect-call like:
-// Ret = (*Foo)(Args);
-// It transforms to:
-// if (Foo == DirectCallee)
-// Ret1 = DirectCallee(Args);
-// else
-// Ret2 = (*Foo)(Args);
-// Ret = phi(Ret1, Ret2);
-// It adds type casts for the args do not match the parameters and the return
-// value. Branch weights metadata also updated.
-// If \p AttachProfToDirectCall is true, a prof metadata is attached to the
-// new direct call to contain \p Count. This is used by SamplePGO inliner to
-// check callsite hotness.
-// Returns the promoted direct call instruction.
-Instruction *llvm::promoteIndirectCall(Instruction *Inst,
- Function *DirectCallee, uint64_t Count,
- uint64_t TotalCount,
- bool AttachProfToDirectCall) {
- assert(DirectCallee != nullptr);
- BasicBlock *BB = Inst->getParent();
- // Just to suppress the non-debug build warning.
- (void)BB;
- DEBUG(dbgs() << "\n\n== Basic Block Before ==\n");
- DEBUG(dbgs() << *BB << "\n");
-
- BasicBlock *DirectCallBB, *IndirectCallBB, *MergeBB;
- createIfThenElse(Inst, DirectCallee, Count, TotalCount, &DirectCallBB,
- &IndirectCallBB, &MergeBB);
Instruction *NewInst =
- createDirectCallInst(Inst, DirectCallee, DirectCallBB, MergeBB);
+ promoteCallWithIfThenElse(CallSite(Inst), DirectCallee, BranchWeights);
if (AttachProfToDirectCall) {
SmallVector<uint32_t, 1> Weights;
Weights.push_back(Count);
MDBuilder MDB(NewInst->getContext());
- dyn_cast<Instruction>(NewInst->stripPointerCasts())
- ->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
- }
-
- // Move Inst from MergeBB to IndirectCallBB.
- Inst->removeFromParent();
- IndirectCallBB->getInstList().insert(IndirectCallBB->getFirstInsertionPt(),
- Inst);
-
- if (InvokeInst *II = dyn_cast<InvokeInst>(Inst)) {
- // At this point, the original indirect invoke instruction has the original
- // UnwindDest and NormalDest. For the direct invoke instruction, the
- // NormalDest points to MergeBB, and MergeBB jumps to the original
- // NormalDest. MergeBB might have a new bitcast instruction for the return
- // value. The PHIs are with the original NormalDest. Since we now have two
- // incoming edges to NormalDest and UnwindDest, we have to do some fixups.
- //
- // UnwindDest will not use the return value. So pass nullptr here.
- fixupPHINodeForUnwind(Inst, II->getUnwindDest(), MergeBB, IndirectCallBB,
- DirectCallBB);
- // We don't need to update the operand from NormalDest for DirectCallBB.
- // Pass nullptr here.
- fixupPHINodeForNormalDest(Inst, II->getNormalDest(), MergeBB,
- IndirectCallBB, NewInst);
+ NewInst->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
}
- insertCallRetPHI(Inst, NewInst, DirectCallee);
+ using namespace ore;
- DEBUG(dbgs() << "\n== Basic Blocks After ==\n");
- DEBUG(dbgs() << *BB << *DirectCallBB << *IndirectCallBB << *MergeBB << "\n");
-
- emitOptimizationRemark(
- BB->getContext(), "pgo-icall-prom", *BB->getParent(), Inst->getDebugLoc(),
- Twine("Promote indirect call to ") + DirectCallee->getName() +
- " with count " + Twine(Count) + " out of " + Twine(TotalCount));
+ if (ORE)
+ ORE->emit([&]() {
+ return OptimizationRemark(DEBUG_TYPE, "Promoted", Inst)
+ << "Promote indirect call to " << NV("DirectCallee", DirectCallee)
+ << " with count " << NV("Count", Count) << " out of "
+ << NV("TotalCount", TotalCount);
+ });
return NewInst;
}
@@ -597,7 +336,8 @@ uint32_t ICallPromotionFunc::tryToPromote(
for (auto &C : Candidates) {
uint64_t Count = C.Count;
- promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, SamplePGO);
+ pgo::promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount,
+ SamplePGO, &ORE);
assert(TotalCount >= Count);
TotalCount -= Count;
NumOfPGOICallPromotion++;
@@ -608,7 +348,7 @@ uint32_t ICallPromotionFunc::tryToPromote(
// Traverse all the indirect-call callsite and get the value profile
// annotation to perform indirect-call promotion.
-bool ICallPromotionFunc::processFunction() {
+bool ICallPromotionFunc::processFunction(ProfileSummaryInfo *PSI) {
bool Changed = false;
ICallPromotionAnalysis ICallAnalysis;
for (auto &I : findIndirectCallSites(F)) {
@@ -616,7 +356,8 @@ bool ICallPromotionFunc::processFunction() {
uint64_t TotalCount;
auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction(
I, NumVals, TotalCount, NumCandidates);
- if (!NumCandidates)
+ if (!NumCandidates ||
+ (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount)))
continue;
auto PromotionCandidates = getPromotionCandidatesForCallSite(
I, ICallProfDataRef, TotalCount, NumCandidates);
@@ -638,7 +379,9 @@ bool ICallPromotionFunc::processFunction() {
}
// A wrapper function that does the actual work.
-static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO) {
+static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI,
+ bool InLTO, bool SamplePGO,
+ ModuleAnalysisManager *AM = nullptr) {
if (DisableICP)
return false;
InstrProfSymtab Symtab;
@@ -654,8 +397,20 @@ static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO) {
continue;
if (F.hasFnAttribute(Attribute::OptimizeNone))
continue;
- ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO);
- bool FuncChanged = ICallPromotion.processFunction();
+
+ std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
+ OptimizationRemarkEmitter *ORE;
+ if (AM) {
+ auto &FAM =
+ AM->getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ ORE = &FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ } else {
+ OwnedORE = llvm::make_unique<OptimizationRemarkEmitter>(&F);
+ ORE = OwnedORE.get();
+ }
+
+ ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO, *ORE);
+ bool FuncChanged = ICallPromotion.processFunction(PSI);
if (ICPDUMPAFTER && FuncChanged) {
DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
DEBUG(dbgs() << "\n");
@@ -670,15 +425,20 @@ static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO) {
}
bool PGOIndirectCallPromotionLegacyPass::runOnModule(Module &M) {
+ ProfileSummaryInfo *PSI =
+ getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+
// Command-line option has the priority for InLTO.
- return promoteIndirectCalls(M, InLTO | ICPLTOMode,
+ return promoteIndirectCalls(M, PSI, InLTO | ICPLTOMode,
SamplePGO | ICPSamplePGOMode);
}
PreservedAnalyses PGOIndirectCallPromotion::run(Module &M,
ModuleAnalysisManager &AM) {
- if (!promoteIndirectCalls(M, InLTO | ICPLTOMode,
- SamplePGO | ICPSamplePGOMode))
+ ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
+
+ if (!promoteIndirectCalls(M, PSI, InLTO | ICPLTOMode,
+ SamplePGO | ICPSamplePGOMode, &AM))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-07 06:38:59 +0000