diff options
Diffstat (limited to 'llvm/lib/Analysis/InlineCost.cpp')
| -rw-r--r-- | llvm/lib/Analysis/InlineCost.cpp | 439 |
1 files changed, 340 insertions, 99 deletions
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index a35f5e11f0e0..4c2413e14435 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -113,6 +113,10 @@ static cl::opt<int> HotCallSiteRelFreq( "entry frequency, for a callsite to be hot in the absence of " "profile information.")); +static cl::opt<int> CallPenalty( + "inline-call-penalty", cl::Hidden, cl::init(25), + cl::desc("Call penalty that is applied per callsite when inlining")); + static cl::opt<bool> OptComputeFullInlineCost( "inline-cost-full", cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::desc("Compute the full inline cost of a call site even when the cost " @@ -390,7 +394,6 @@ protected: bool visitPtrToInt(PtrToIntInst &I); bool visitIntToPtr(IntToPtrInst &I); bool visitCastInst(CastInst &I); - bool visitUnaryInstruction(UnaryInstruction &I); bool visitCmpInst(CmpInst &I); bool visitSub(BinaryOperator &I); bool visitBinaryOperator(BinaryOperator &I); @@ -411,19 +414,18 @@ protected: bool visitUnreachableInst(UnreachableInst &I); public: - CallAnalyzer( - Function &Callee, CallBase &Call, const TargetTransformInfo &TTI, - function_ref<AssumptionCache &(Function &)> GetAssumptionCache, - function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr, - ProfileSummaryInfo *PSI = nullptr, - OptimizationRemarkEmitter *ORE = nullptr) + CallAnalyzer(Function &Callee, CallBase &Call, const TargetTransformInfo &TTI, + function_ref<AssumptionCache &(Function &)> GetAssumptionCache, + function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr, + ProfileSummaryInfo *PSI = nullptr, + OptimizationRemarkEmitter *ORE = nullptr) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI), PSI(PSI), F(Callee), DL(F.getParent()->getDataLayout()), ORE(ORE), CandidateCall(Call), EnableLoadElimination(true) {} InlineResult analyze(); - Optional<Constant*> getSimplifiedValue(Instruction *I) { + Optional<Constant *> getSimplifiedValue(Instruction *I) { if (SimplifiedValues.find(I) != SimplifiedValues.end()) return SimplifiedValues[I]; return None; @@ -441,6 +443,25 @@ public: void dump(); }; +// Considering forming a binary search, we should find the number of nodes +// which is same as the number of comparisons when lowered. For a given +// number of clusters, n, we can define a recursive function, f(n), to find +// the number of nodes in the tree. The recursion is : +// f(n) = 1 + f(n/2) + f (n - n/2), when n > 3, +// and f(n) = n, when n <= 3. +// This will lead a binary tree where the leaf should be either f(2) or f(3) +// when n > 3. So, the number of comparisons from leaves should be n, while +// the number of non-leaf should be : +// 2^(log2(n) - 1) - 1 +// = 2^log2(n) * 2^-1 - 1 +// = n / 2 - 1. +// Considering comparisons from leaf and non-leaf nodes, we can estimate the +// number of comparisons in a simple closed form : +// n + n / 2 - 1 = n * 3 / 2 - 1 +int64_t getExpectedNumberOfCompare(int NumCaseCluster) { + return 3 * static_cast<int64_t>(NumCaseCluster) / 2 - 1; +} + /// FIXME: if it is necessary to derive from InlineCostCallAnalyzer, note /// the FIXME in onLoweredCall, when instantiating an InlineCostCallAnalyzer class InlineCostCallAnalyzer final : public CallAnalyzer { @@ -489,6 +510,12 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // sense that it's not weighted by profile counts at all. int ColdSize = 0; + // Whether inlining is decided by cost-benefit analysis. + bool DecidedByCostBenefit = false; + + // The cost-benefit pair computed by cost-benefit analysis. + Optional<CostBenefitPair> CostBenefit = None; + bool SingleBB = true; unsigned SROACostSavings = 0; @@ -514,7 +541,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { /// Handle a capped 'int' increment for Cost. void addCost(int64_t Inc, int64_t UpperBound = INT_MAX) { assert(UpperBound > 0 && UpperBound <= INT_MAX && "invalid upper bound"); - Cost = (int)std::min(UpperBound, Cost + Inc); + Cost = std::min<int>(UpperBound, Cost + Inc); } void onDisableSROA(AllocaInst *Arg) override { @@ -531,7 +558,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { addCost(LoadEliminationCost); LoadEliminationCost = 0; } - void onCallPenalty() override { addCost(InlineConstants::CallPenalty); } + void onCallPenalty() override { addCost(CallPenalty); } void onCallArgumentSetup(const CallBase &Call) override { // Pay the price of the argument setup. We account for the average 1 // instruction per call argument setup here. @@ -566,7 +593,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { } } else // Otherwise simply add the cost for merely making the call. - addCost(InlineConstants::CallPenalty); + addCost(CallPenalty); } void onFinalizeSwitch(unsigned JumpTableSize, @@ -575,38 +602,26 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // branch to destination. // Maximum valid cost increased in this function. if (JumpTableSize) { - int64_t JTCost = (int64_t)JumpTableSize * InlineConstants::InstrCost + - 4 * InlineConstants::InstrCost; + int64_t JTCost = + static_cast<int64_t>(JumpTableSize) * InlineConstants::InstrCost + + 4 * InlineConstants::InstrCost; - addCost(JTCost, (int64_t)CostUpperBound); + addCost(JTCost, static_cast<int64_t>(CostUpperBound)); return; } - // Considering forming a binary search, we should find the number of nodes - // which is same as the number of comparisons when lowered. For a given - // number of clusters, n, we can define a recursive function, f(n), to find - // the number of nodes in the tree. The recursion is : - // f(n) = 1 + f(n/2) + f (n - n/2), when n > 3, - // and f(n) = n, when n <= 3. - // This will lead a binary tree where the leaf should be either f(2) or f(3) - // when n > 3. So, the number of comparisons from leaves should be n, while - // the number of non-leaf should be : - // 2^(log2(n) - 1) - 1 - // = 2^log2(n) * 2^-1 - 1 - // = n / 2 - 1. - // Considering comparisons from leaf and non-leaf nodes, we can estimate the - // number of comparisons in a simple closed form : - // n + n / 2 - 1 = n * 3 / 2 - 1 + if (NumCaseCluster <= 3) { // Suppose a comparison includes one compare and one conditional branch. addCost(NumCaseCluster * 2 * InlineConstants::InstrCost); return; } - int64_t ExpectedNumberOfCompare = 3 * (int64_t)NumCaseCluster / 2 - 1; + int64_t ExpectedNumberOfCompare = + getExpectedNumberOfCompare(NumCaseCluster); int64_t SwitchCost = ExpectedNumberOfCompare * 2 * InlineConstants::InstrCost; - addCost(SwitchCost, (int64_t)CostUpperBound); + addCost(SwitchCost, static_cast<int64_t>(CostUpperBound)); } void onMissedSimplification() override { addCost(InlineConstants::InstrCost); @@ -672,15 +687,22 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { } bool isCostBenefitAnalysisEnabled() { - if (!InlineEnableCostBenefitAnalysis) - return false; - if (!PSI || !PSI->hasProfileSummary()) return false; if (!GetBFI) return false; + if (InlineEnableCostBenefitAnalysis.getNumOccurrences()) { + // Honor the explicit request from the user. + if (!InlineEnableCostBenefitAnalysis) + return false; + } else { + // Otherwise, require instrumentation profile. + if (!PSI->hasInstrumentationProfile()) + return false; + } + auto *Caller = CandidateCall.getParent()->getParent(); if (!Caller->getEntryCount()) return false; @@ -693,7 +715,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { if (!PSI->isHotCallSite(CandidateCall, CallerBFI)) return false; - if (!F.getEntryCount()) + // Make sure we have a nonzero entry count. + auto EntryCount = F.getEntryCount(); + if (!EntryCount || !EntryCount.getCount()) return false; BlockFrequencyInfo *CalleeBFI = &(GetBFI(F)); @@ -749,9 +773,6 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { CurrentSavings += InlineConstants::InstrCost; } } - // TODO: Consider other forms of savings like switch statements, - // indirect calls becoming direct, SROACostSavings, LoadEliminationCost, - // etc. } auto ProfileCount = CalleeBFI->getBlockProfileCount(&BB); @@ -762,7 +783,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // Compute the cycle savings per call. auto EntryProfileCount = F.getEntryCount(); - assert(EntryProfileCount.hasValue()); + assert(EntryProfileCount.hasValue() && EntryProfileCount.getCount()); auto EntryCount = EntryProfileCount.getCount(); CycleSavings += EntryCount / 2; CycleSavings = CycleSavings.udiv(EntryCount); @@ -780,6 +801,8 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // savings threshold. Size = Size > InlineSizeAllowance ? Size - InlineSizeAllowance : 1; + CostBenefit.emplace(APInt(128, Size), CycleSavings); + // Return true if the savings justify the cost of inlining. Specifically, // we evaluate the following inequality: // @@ -813,7 +836,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { continue; NumLoops++; } - addCost(NumLoops * InlineConstants::CallPenalty); + addCost(NumLoops * InlineConstants::LoopPenalty); } // We applied the maximum possible vector bonus at the beginning. Now, @@ -825,6 +848,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { Threshold -= VectorBonus / 2; if (auto Result = costBenefitAnalysis()) { + DecidedByCostBenefit = true; if (Result.getValue()) return InlineResult::success(); else @@ -924,9 +948,213 @@ public: } virtual ~InlineCostCallAnalyzer() {} - int getThreshold() { return Threshold; } - int getCost() { return Cost; } + int getThreshold() const { return Threshold; } + int getCost() const { return Cost; } + Optional<CostBenefitPair> getCostBenefitPair() { return CostBenefit; } + bool wasDecidedByCostBenefit() const { return DecidedByCostBenefit; } +}; + +class InlineCostFeaturesAnalyzer final : public CallAnalyzer { +private: + InlineCostFeatures Cost = {}; + + // FIXME: These constants are taken from the heuristic-based cost visitor. + // These should be removed entirely in a later revision to avoid reliance on + // heuristics in the ML inliner. + static constexpr int JTCostMultiplier = 4; + static constexpr int CaseClusterCostMultiplier = 2; + static constexpr int SwitchCostMultiplier = 2; + + // FIXME: These are taken from the heuristic-based cost visitor: we should + // eventually abstract these to the CallAnalyzer to avoid duplication. + unsigned SROACostSavingOpportunities = 0; + int VectorBonus = 0; + int SingleBBBonus = 0; + int Threshold = 5; + + DenseMap<AllocaInst *, unsigned> SROACosts; + + void increment(InlineCostFeatureIndex Feature, int64_t Delta = 1) { + Cost[static_cast<size_t>(Feature)] += Delta; + } + + void set(InlineCostFeatureIndex Feature, int64_t Value) { + Cost[static_cast<size_t>(Feature)] = Value; + } + + void onDisableSROA(AllocaInst *Arg) override { + auto CostIt = SROACosts.find(Arg); + if (CostIt == SROACosts.end()) + return; + + increment(InlineCostFeatureIndex::SROALosses, CostIt->second); + SROACostSavingOpportunities -= CostIt->second; + SROACosts.erase(CostIt); + } + + void onDisableLoadElimination() override { + set(InlineCostFeatureIndex::LoadElimination, 1); + } + + void onCallPenalty() override { + increment(InlineCostFeatureIndex::CallPenalty, CallPenalty); + } + + void onCallArgumentSetup(const CallBase &Call) override { + increment(InlineCostFeatureIndex::CallArgumentSetup, + Call.arg_size() * InlineConstants::InstrCost); + } + + void onLoadRelativeIntrinsic() override { + increment(InlineCostFeatureIndex::LoadRelativeIntrinsic, + 3 * InlineConstants::InstrCost); + } + + void onLoweredCall(Function *F, CallBase &Call, + bool IsIndirectCall) override { + increment(InlineCostFeatureIndex::LoweredCallArgSetup, + Call.arg_size() * InlineConstants::InstrCost); + + if (IsIndirectCall) { + InlineParams IndirectCallParams = {/* DefaultThreshold*/ 0, + /*HintThreshold*/ {}, + /*ColdThreshold*/ {}, + /*OptSizeThreshold*/ {}, + /*OptMinSizeThreshold*/ {}, + /*HotCallSiteThreshold*/ {}, + /*LocallyHotCallSiteThreshold*/ {}, + /*ColdCallSiteThreshold*/ {}, + /*ComputeFullInlineCost*/ true, + /*EnableDeferral*/ true}; + IndirectCallParams.DefaultThreshold = + InlineConstants::IndirectCallThreshold; + + InlineCostCallAnalyzer CA(*F, Call, IndirectCallParams, TTI, + GetAssumptionCache, GetBFI, PSI, ORE, false, + true); + if (CA.analyze().isSuccess()) { + increment(InlineCostFeatureIndex::NestedInlineCostEstimate, + CA.getCost()); + increment(InlineCostFeatureIndex::NestedInlines, 1); + } + } else { + onCallPenalty(); + } + } + + void onFinalizeSwitch(unsigned JumpTableSize, + unsigned NumCaseCluster) override { + + if (JumpTableSize) { + int64_t JTCost = + static_cast<int64_t>(JumpTableSize) * InlineConstants::InstrCost + + JTCostMultiplier * InlineConstants::InstrCost; + increment(InlineCostFeatureIndex::JumpTablePenalty, JTCost); + return; + } + + if (NumCaseCluster <= 3) { + increment(InlineCostFeatureIndex::CaseClusterPenalty, + NumCaseCluster * CaseClusterCostMultiplier * + InlineConstants::InstrCost); + return; + } + + int64_t ExpectedNumberOfCompare = + getExpectedNumberOfCompare(NumCaseCluster); + + int64_t SwitchCost = ExpectedNumberOfCompare * SwitchCostMultiplier * + InlineConstants::InstrCost; + increment(InlineCostFeatureIndex::SwitchPenalty, SwitchCost); + } + + void onMissedSimplification() override { + increment(InlineCostFeatureIndex::UnsimplifiedCommonInstructions, + InlineConstants::InstrCost); + } + + void onInitializeSROAArg(AllocaInst *Arg) override { SROACosts[Arg] = 0; } + void onAggregateSROAUse(AllocaInst *Arg) override { + SROACosts.find(Arg)->second += InlineConstants::InstrCost; + SROACostSavingOpportunities += InlineConstants::InstrCost; + } + + void onBlockAnalyzed(const BasicBlock *BB) override { + if (BB->getTerminator()->getNumSuccessors() > 1) + set(InlineCostFeatureIndex::IsMultipleBlocks, 1); + Threshold -= SingleBBBonus; + } + + InlineResult finalizeAnalysis() override { + auto *Caller = CandidateCall.getFunction(); + if (Caller->hasMinSize()) { + DominatorTree DT(F); + LoopInfo LI(DT); + for (Loop *L : LI) { + // Ignore loops that will not be executed + if (DeadBlocks.count(L->getHeader())) + continue; + increment(InlineCostFeatureIndex::NumLoops, + InlineConstants::LoopPenalty); + } + } + set(InlineCostFeatureIndex::DeadBlocks, DeadBlocks.size()); + set(InlineCostFeatureIndex::SimplifiedInstructions, + NumInstructionsSimplified); + set(InlineCostFeatureIndex::ConstantArgs, NumConstantArgs); + set(InlineCostFeatureIndex::ConstantOffsetPtrArgs, + NumConstantOffsetPtrArgs); + set(InlineCostFeatureIndex::SROASavings, SROACostSavingOpportunities); + + if (NumVectorInstructions <= NumInstructions / 10) + Threshold -= VectorBonus; + else if (NumVectorInstructions <= NumInstructions / 2) + Threshold -= VectorBonus / 2; + + set(InlineCostFeatureIndex::Threshold, Threshold); + + return InlineResult::success(); + } + + bool shouldStop() override { return false; } + + void onLoadEliminationOpportunity() override { + increment(InlineCostFeatureIndex::LoadElimination, 1); + } + + InlineResult onAnalysisStart() override { + increment(InlineCostFeatureIndex::CallSiteCost, + -1 * getCallsiteCost(this->CandidateCall, DL)); + + set(InlineCostFeatureIndex::ColdCcPenalty, + (F.getCallingConv() == CallingConv::Cold)); + + // FIXME: we shouldn't repeat this logic in both the Features and Cost + // analyzer - instead, we should abstract it to a common method in the + // CallAnalyzer + int SingleBBBonusPercent = 50; + int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); + Threshold += TTI.adjustInliningThreshold(&CandidateCall); + Threshold *= TTI.getInliningThresholdMultiplier(); + SingleBBBonus = Threshold * SingleBBBonusPercent / 100; + VectorBonus = Threshold * VectorBonusPercent / 100; + Threshold += (SingleBBBonus + VectorBonus); + + return InlineResult::success(); + } + +public: + InlineCostFeaturesAnalyzer( + const TargetTransformInfo &TTI, + function_ref<AssumptionCache &(Function &)> &GetAssumptionCache, + function_ref<BlockFrequencyInfo &(Function &)> GetBFI, + ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE, Function &Callee, + CallBase &Call) + : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, PSI) {} + + const InlineCostFeatures &features() const { return Cost; } }; + } // namespace /// Test whether the given value is an Alloca-derived function argument. @@ -940,8 +1168,8 @@ void CallAnalyzer::disableSROAForArg(AllocaInst *SROAArg) { disableLoadElimination(); } -void InlineCostAnnotationWriter::emitInstructionAnnot(const Instruction *I, - formatted_raw_ostream &OS) { +void InlineCostAnnotationWriter::emitInstructionAnnot( + const Instruction *I, formatted_raw_ostream &OS) { // The cost of inlining of the given instruction is printed always. // The threshold delta is printed only when it is non-zero. It happens // when we decided to give a bonus at a particular instruction. @@ -1023,12 +1251,14 @@ bool CallAnalyzer::isGEPFree(GetElementPtrInst &GEP) { Operands.push_back(SimpleOp); else Operands.push_back(Op); - return TargetTransformInfo::TCC_Free == - TTI.getUserCost(&GEP, Operands, - TargetTransformInfo::TCK_SizeAndLatency); + return TTI.getUserCost(&GEP, Operands, + TargetTransformInfo::TCK_SizeAndLatency) == + TargetTransformInfo::TCC_Free; } bool CallAnalyzer::visitAlloca(AllocaInst &I) { + disableSROA(I.getOperand(0)); + // Check whether inlining will turn a dynamic alloca into a static // alloca and handle that case. if (I.isArrayAllocation()) { @@ -1044,13 +1274,11 @@ bool CallAnalyzer::visitAlloca(AllocaInst &I) { // is needed to track stack usage during inlining. Type *Ty = I.getAllocatedType(); AllocatedSize = SaturatingMultiplyAdd( - AllocSize->getLimitedValue(), DL.getTypeAllocSize(Ty).getKnownMinSize(), - AllocatedSize); - if (AllocatedSize > InlineConstants::MaxSimplifiedDynamicAllocaToInline) { + AllocSize->getLimitedValue(), + DL.getTypeAllocSize(Ty).getKnownMinSize(), AllocatedSize); + if (AllocatedSize > InlineConstants::MaxSimplifiedDynamicAllocaToInline) HasDynamicAlloca = true; - return false; - } - return Base::visitAlloca(I); + return false; } } @@ -1061,15 +1289,13 @@ bool CallAnalyzer::visitAlloca(AllocaInst &I) { SaturatingAdd(DL.getTypeAllocSize(Ty).getKnownMinSize(), AllocatedSize); } - // We will happily inline static alloca instructions. - if (I.isStaticAlloca()) - return Base::visitAlloca(I); - // FIXME: This is overly conservative. Dynamic allocas are inefficient for // a variety of reasons, and so we would like to not inline them into // functions which don't currently have a dynamic alloca. This simply // disables inlining altogether in the presence of a dynamic alloca. - HasDynamicAlloca = true; + if (!I.isStaticAlloca()) + HasDynamicAlloca = true; + return false; } @@ -1202,11 +1428,11 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) { if (!DisableGEPConstOperand) if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - SmallVector<Constant *, 2> Indices; - for (unsigned int Index = 1 ; Index < COps.size() ; ++Index) + SmallVector<Constant *, 2> Indices; + for (unsigned int Index = 1; Index < COps.size(); ++Index) Indices.push_back(COps[Index]); - return ConstantExpr::getGetElementPtr(I.getSourceElementType(), COps[0], - Indices, I.isInBounds()); + return ConstantExpr::getGetElementPtr( + I.getSourceElementType(), COps[0], Indices, I.isInBounds()); })) return true; @@ -1295,8 +1521,8 @@ bool CallAnalyzer::visitPtrToInt(PtrToIntInst &I) { if (auto *SROAArg = getSROAArgForValueOrNull(I.getOperand(0))) SROAArgValues[&I] = SROAArg; - return TargetTransformInfo::TCC_Free == - TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); + return TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) == + TargetTransformInfo::TCC_Free; } bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { @@ -1320,8 +1546,8 @@ bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { if (auto *SROAArg = getSROAArgForValueOrNull(Op)) SROAArgValues[&I] = SROAArg; - return TargetTransformInfo::TCC_Free == - TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); + return TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) == + TargetTransformInfo::TCC_Free; } bool CallAnalyzer::visitCastInst(CastInst &I) { @@ -1352,21 +1578,8 @@ bool CallAnalyzer::visitCastInst(CastInst &I) { break; } - return TargetTransformInfo::TCC_Free == - TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); -} - -bool CallAnalyzer::visitUnaryInstruction(UnaryInstruction &I) { - Value *Operand = I.getOperand(0); - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantFoldInstOperands(&I, COps[0], DL); - })) - return true; - - // Disable any SROA on the argument to arbitrary unary instructions. - disableSROA(Operand); - - return false; + return TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) == + TargetTransformInfo::TCC_Free; } bool CallAnalyzer::paramHasAttr(Argument *A, Attribute::AttrKind Attr) { @@ -1577,10 +1790,11 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { } } + Threshold += TTI.adjustInliningThreshold(&Call); + // Finally, take the target-specific inlining threshold multiplier into // account. Threshold *= TTI.getInliningThresholdMultiplier(); - Threshold += TTI.adjustInliningThreshold(&Call); SingleBBBonus = Threshold * SingleBBBonusPercent / 100; VectorBonus = Threshold * VectorBonusPercent / 100; @@ -1763,8 +1977,8 @@ bool CallAnalyzer::visitExtractValue(ExtractValueInst &I) { })) return true; - // SROA can look through these but give them a cost. - return false; + // SROA can't look through these, but they may be free. + return Base::visitExtractValue(I); } bool CallAnalyzer::visitInsertValue(InsertValueInst &I) { @@ -1776,8 +1990,8 @@ bool CallAnalyzer::visitInsertValue(InsertValueInst &I) { })) return true; - // SROA can look through these but give them a cost. - return false; + // SROA can't look through these, but they may be free. + return Base::visitInsertValue(I); } /// Try to simplify a call site. @@ -1872,6 +2086,11 @@ bool CallAnalyzer::visitCallBase(CallBase &Call) { case Intrinsic::vastart: InitsVargArgs = true; return false; + case Intrinsic::launder_invariant_group: + case Intrinsic::strip_invariant_group: + if (auto *SROAArg = getSROAArgForValueOrNull(II->getOperand(0))) + SROAArgValues[II] = SROAArg; + return true; } } @@ -1948,9 +2167,9 @@ bool CallAnalyzer::visitSelectInst(SelectInst &SI) { } // Select condition is a constant. - Value *SelectedV = CondC->isAllOnesValue() - ? TrueVal - : (CondC->isNullValue()) ? FalseVal : nullptr; + Value *SelectedV = CondC->isAllOnesValue() ? TrueVal + : (CondC->isNullValue()) ? FalseVal + : nullptr; if (!SelectedV) { // Condition is a vector constant that is not all 1s or all 0s. If all // operands are constants, ConstantExpr::getSelect() can handle the cases @@ -2002,7 +2221,7 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { // proportional to the size of the tree or the size of jump table range. // // NB: We convert large switches which are just used to initialize large phi - // nodes to lookup tables instead in simplify-cfg, so this shouldn't prevent + // nodes to lookup tables instead in simplifycfg, so this shouldn't prevent // inlining those. It will prevent inlining in cases where the optimization // does not (yet) fire. @@ -2056,8 +2275,8 @@ bool CallAnalyzer::visitUnreachableInst(UnreachableInst &I) { bool CallAnalyzer::visitInstruction(Instruction &I) { // Some instructions are free. All of the free intrinsics can also be // handled by SROA, etc. - if (TargetTransformInfo::TCC_Free == - TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency)) + if (TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) == + TargetTransformInfo::TCC_Free) return true; // We found something we don't understand or can't handle. Mark any SROA-able @@ -2415,9 +2634,7 @@ void InlineCostCallAnalyzer::print() { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Dump stats about this call's analysis. -LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() { - print(); -} +LLVM_DUMP_METHOD void InlineCostCallAnalyzer::dump() { print(); } #endif /// Test that there are no attribute conflicts between Caller and Callee @@ -2443,7 +2660,7 @@ int llvm::getCallsiteCost(CallBase &Call, const DataLayout &DL) { // We approximate the number of loads and stores needed by dividing the // size of the byval type by the target's pointer size. PointerType *PTy = cast<PointerType>(Call.getArgOperand(I)->getType()); - unsigned TypeSize = DL.getTypeSizeInBits(PTy->getElementType()); + unsigned TypeSize = DL.getTypeSizeInBits(Call.getParamByValType(I)); unsigned AS = PTy->getAddressSpace(); unsigned PointerSize = DL.getPointerSizeInBits(AS); // Ceiling division. @@ -2465,7 +2682,7 @@ int llvm::getCallsiteCost(CallBase &Call, const DataLayout &DL) { } } // The call instruction also disappears after inlining. - Cost += InlineConstants::InstrCost + InlineConstants::CallPenalty; + Cost += InlineConstants::InstrCost + CallPenalty; return Cost; } @@ -2504,6 +2721,19 @@ Optional<int> llvm::getInliningCostEstimate( return CA.getCost(); } +Optional<InlineCostFeatures> llvm::getInliningCostFeatures( + CallBase &Call, TargetTransformInfo &CalleeTTI, + function_ref<AssumptionCache &(Function &)> GetAssumptionCache, + function_ref<BlockFrequencyInfo &(Function &)> GetBFI, + ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) { + InlineCostFeaturesAnalyzer CFA(CalleeTTI, GetAssumptionCache, GetBFI, PSI, + ORE, *Call.getCalledFunction(), Call); + auto R = CFA.analyze(); + if (!R.isSuccess()) + return None; + return CFA.features(); +} + Optional<InlineResult> llvm::getAttributeBasedInliningDecision( CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref<const TargetLibraryInfo &(Function &)> GetTLI) { @@ -2608,6 +2838,17 @@ InlineCost llvm::getInlineCost( LLVM_DEBUG(CA.dump()); + // Always make cost benefit based decision explicit. + // We use always/never here since threshold is not meaningful, + // as it's not what drives cost-benefit analysis. + if (CA.wasDecidedByCostBenefit()) { + if (ShouldInline.isSuccess()) + return InlineCost::getAlways("benefit over cost", + CA.getCostBenefitPair()); + else + return InlineCost::getNever("cost over benefit", CA.getCostBenefitPair()); + } + // Check if there was a reason to force inlining or no inlining. if (!ShouldInline.isSuccess() && CA.getCost() < CA.getThreshold()) return InlineCost::getNever(ShouldInline.getFailureReason()); @@ -2761,8 +3002,8 @@ PreservedAnalyses InlineCostAnnotationPrinterPass::run(Function &F, FunctionAnalysisManager &FAM) { PrintInstructionComments = true; - std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&]( - Function &F) -> AssumptionCache & { + std::function<AssumptionCache &(Function &)> GetAssumptionCache = + [&](Function &F) -> AssumptionCache & { return FAM.getResult<AssumptionAnalysis>(F); }; Module *M = F.getParent(); |