aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp208
1 files changed, 86 insertions, 122 deletions
diff --git a/contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 143715006512..ee382657f5e6 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -15,7 +15,6 @@
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
@@ -49,10 +48,6 @@ using namespace llvm;
STATISTIC(NumInlined, "Number of functions inlined");
STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
-static cl::opt<bool> InlineEnablePriorityOrder(
- "module-inline-enable-priority-order", cl::Hidden, cl::init(true),
- cl::desc("Enable the priority inline order for the module inliner"));
-
/// Return true if the specified inline history ID
/// indicates an inline history that includes the specified function.
static bool inlineHistoryIncludes(
@@ -85,8 +80,7 @@ InlineAdvisor &ModuleInlinerPass::getAdvisor(const ModuleAnalysisManager &MAM,
// would get from the MAM can be invalidated as a result of the inliner's
// activity.
OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
- M, FAM, Params,
- InlineContext{LTOPhase, InlinePass::ModuleInliner});
+ M, FAM, Params, InlineContext{LTOPhase, InlinePass::ModuleInliner});
return *OwnedAdvisor;
}
@@ -111,9 +105,8 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
LLVM_DEBUG(dbgs() << "---- Module Inliner is Running ---- \n");
auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
- if (!IAA.tryCreate(
- Params, Mode, {},
- InlineContext{LTOPhase, InlinePass::ModuleInliner})) {
+ if (!IAA.tryCreate(Params, Mode, {},
+ InlineContext{LTOPhase, InlinePass::ModuleInliner})) {
M.getContext().emitError(
"Could not setup Inlining Advisor for the requested "
"mode and/or options");
@@ -145,12 +138,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
//
// TODO: Here is a huge amount duplicate code between the module inliner and
// the SCC inliner, which need some refactoring.
- std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>> Calls;
- if (InlineEnablePriorityOrder)
- Calls = std::make_unique<PriorityInlineOrder>(
- std::make_unique<SizePriority>());
- else
- Calls = std::make_unique<DefaultInlineOrder<std::pair<CallBase *, int>>>();
+ auto Calls = getInlineOrder(FAM, Params);
assert(Calls != nullptr && "Expected an initialized InlineOrder");
// Populate the initial list of calls in this module.
@@ -188,135 +176,111 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
// index into the InlineHistory vector.
SmallVector<std::pair<Function *, int>, 16> InlineHistory;
- // Track a set vector of inlined callees so that we can augment the caller
- // with all of their edges in the call graph before pruning out the ones that
- // got simplified away.
- SmallSetVector<Function *, 4> InlinedCallees;
-
// Track the dead functions to delete once finished with inlining calls. We
// defer deleting these to make it easier to handle the call graph updates.
SmallVector<Function *, 4> DeadFunctions;
// Loop forward over all of the calls.
while (!Calls->empty()) {
- // We expect the calls to typically be batched with sequences of calls that
- // have the same caller, so we first set up some shared infrastructure for
- // this caller. We also do any pruning we can at this layer on the caller
- // alone.
- Function &F = *Calls->front().first->getCaller();
+ auto P = Calls->pop();
+ CallBase *CB = P.first;
+ const int InlineHistoryID = P.second;
+ Function &F = *CB->getCaller();
+ Function &Callee = *CB->getCalledFunction();
LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
<< " Function size: " << F.getInstructionCount()
<< "\n");
+ (void)F;
auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
return FAM.getResult<AssumptionAnalysis>(F);
};
- // Now process as many calls as we have within this caller in the sequence.
- // We bail out as soon as the caller has to change so we can
- // prepare the context of that new caller.
- bool DidInline = false;
- while (!Calls->empty() && Calls->front().first->getCaller() == &F) {
- auto P = Calls->pop();
- CallBase *CB = P.first;
- const int InlineHistoryID = P.second;
- Function &Callee = *CB->getCalledFunction();
-
- if (InlineHistoryID != -1 &&
- inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
- setInlineRemark(*CB, "recursive");
- continue;
- }
+ if (InlineHistoryID != -1 &&
+ inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
+ setInlineRemark(*CB, "recursive");
+ continue;
+ }
- auto Advice = Advisor.getAdvice(*CB, /*OnlyMandatory*/ false);
- // Check whether we want to inline this callsite.
- if (!Advice->isInliningRecommended()) {
- Advice->recordUnattemptedInlining();
- continue;
- }
+ auto Advice = Advisor.getAdvice(*CB, /*OnlyMandatory*/ false);
+ // Check whether we want to inline this callsite.
+ if (!Advice->isInliningRecommended()) {
+ Advice->recordUnattemptedInlining();
+ continue;
+ }
- // Setup the data structure used to plumb customization into the
- // `InlineFunction` routine.
- InlineFunctionInfo IFI(
- /*cg=*/nullptr, GetAssumptionCache, PSI,
- &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
- &FAM.getResult<BlockFrequencyAnalysis>(Callee));
-
- InlineResult IR =
- InlineFunction(*CB, IFI, &FAM.getResult<AAManager>(*CB->getCaller()));
- if (!IR.isSuccess()) {
- Advice->recordUnsuccessfulInlining(IR);
- continue;
- }
+ // Setup the data structure used to plumb customization into the
+ // `InlineFunction` routine.
+ InlineFunctionInfo IFI(
+ /*cg=*/nullptr, GetAssumptionCache, PSI,
+ &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
+ &FAM.getResult<BlockFrequencyAnalysis>(Callee));
+
+ InlineResult IR =
+ InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
+ &FAM.getResult<AAManager>(*CB->getCaller()));
+ if (!IR.isSuccess()) {
+ Advice->recordUnsuccessfulInlining(IR);
+ continue;
+ }
- DidInline = true;
- InlinedCallees.insert(&Callee);
- ++NumInlined;
-
- LLVM_DEBUG(dbgs() << " Size after inlining: "
- << F.getInstructionCount() << "\n");
-
- // Add any new callsites to defined functions to the worklist.
- if (!IFI.InlinedCallSites.empty()) {
- int NewHistoryID = InlineHistory.size();
- InlineHistory.push_back({&Callee, InlineHistoryID});
-
- for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
- Function *NewCallee = ICB->getCalledFunction();
- if (!NewCallee) {
- // Try to promote an indirect (virtual) call without waiting for
- // the post-inline cleanup and the next DevirtSCCRepeatedPass
- // iteration because the next iteration may not happen and we may
- // miss inlining it.
- if (tryPromoteCall(*ICB))
- NewCallee = ICB->getCalledFunction();
- }
- if (NewCallee)
- if (!NewCallee->isDeclaration())
- Calls->push({ICB, NewHistoryID});
- }
- }
+ Changed = true;
+ ++NumInlined;
+
+ LLVM_DEBUG(dbgs() << " Size after inlining: " << F.getInstructionCount()
+ << "\n");
- // Merge the attributes based on the inlining.
- AttributeFuncs::mergeAttributesForInlining(F, Callee);
-
- // For local functions, check whether this makes the callee trivially
- // dead. In that case, we can drop the body of the function eagerly
- // which may reduce the number of callers of other functions to one,
- // changing inline cost thresholds.
- bool CalleeWasDeleted = false;
- if (Callee.hasLocalLinkage()) {
- // To check this we also need to nuke any dead constant uses (perhaps
- // made dead by this operation on other functions).
- Callee.removeDeadConstantUsers();
- // if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
- if (Callee.use_empty() && !isKnownLibFunction(Callee, GetTLI(Callee))) {
- Calls->erase_if([&](const std::pair<CallBase *, int> &Call) {
- return Call.first->getCaller() == &Callee;
- });
- // Clear the body and queue the function itself for deletion when we
- // finish inlining.
- // Note that after this point, it is an error to do anything other
- // than use the callee's address or delete it.
- Callee.dropAllReferences();
- assert(!is_contained(DeadFunctions, &Callee) &&
- "Cannot put cause a function to become dead twice!");
- DeadFunctions.push_back(&Callee);
- CalleeWasDeleted = true;
+ // Add any new callsites to defined functions to the worklist.
+ if (!IFI.InlinedCallSites.empty()) {
+ int NewHistoryID = InlineHistory.size();
+ InlineHistory.push_back({&Callee, InlineHistoryID});
+
+ for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
+ Function *NewCallee = ICB->getCalledFunction();
+ if (!NewCallee) {
+ // Try to promote an indirect (virtual) call without waiting for
+ // the post-inline cleanup and the next DevirtSCCRepeatedPass
+ // iteration because the next iteration may not happen and we may
+ // miss inlining it.
+ if (tryPromoteCall(*ICB))
+ NewCallee = ICB->getCalledFunction();
}
+ if (NewCallee)
+ if (!NewCallee->isDeclaration())
+ Calls->push({ICB, NewHistoryID});
}
- if (CalleeWasDeleted)
- Advice->recordInliningWithCalleeDeleted();
- else
- Advice->recordInlining();
}
- if (!DidInline)
- continue;
- Changed = true;
-
- InlinedCallees.clear();
+ // For local functions, check whether this makes the callee trivially
+ // dead. In that case, we can drop the body of the function eagerly
+ // which may reduce the number of callers of other functions to one,
+ // changing inline cost thresholds.
+ bool CalleeWasDeleted = false;
+ if (Callee.hasLocalLinkage()) {
+ // To check this we also need to nuke any dead constant uses (perhaps
+ // made dead by this operation on other functions).
+ Callee.removeDeadConstantUsers();
+ // if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
+ if (Callee.use_empty() && !isKnownLibFunction(Callee, GetTLI(Callee))) {
+ Calls->erase_if([&](const std::pair<CallBase *, int> &Call) {
+ return Call.first->getCaller() == &Callee;
+ });
+ // Clear the body and queue the function itself for deletion when we
+ // finish inlining.
+ // Note that after this point, it is an error to do anything other
+ // than use the callee's address or delete it.
+ Callee.dropAllReferences();
+ assert(!is_contained(DeadFunctions, &Callee) &&
+ "Cannot put cause a function to become dead twice!");
+ DeadFunctions.push_back(&Callee);
+ CalleeWasDeleted = true;
+ }
+ }
+ if (CalleeWasDeleted)
+ Advice->recordInliningWithCalleeDeleted();
+ else
+ Advice->recordInlining();
}
// Now that we've finished inlining all of the calls across this module,
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-13 03:50:16 +0000