summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp195
1 files changed, 162 insertions, 33 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp b/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp
index af556e2c856e..7d64828aa482 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp
@@ -19,6 +19,7 @@
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator_range.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
@@ -35,7 +36,9 @@
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/DebugLoc.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/BranchProbability.h"
@@ -211,6 +214,7 @@ namespace {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequired<MachineBranchProbabilityInfo>();
+ AU.addRequired<ProfileSummaryInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -285,14 +289,113 @@ namespace {
Prediction);
}
- bool MeetIfcvtSizeLimit(MachineBasicBlock &TBB,
- unsigned TCycle, unsigned TExtra,
- MachineBasicBlock &FBB,
- unsigned FCycle, unsigned FExtra,
- BranchProbability Prediction) const {
- return TCycle > 0 && FCycle > 0 &&
- TII->isProfitableToIfCvt(TBB, TCycle, TExtra, FBB, FCycle, FExtra,
- Prediction);
+ bool MeetIfcvtSizeLimit(BBInfo &TBBInfo, BBInfo &FBBInfo,
+ MachineBasicBlock &CommBB, unsigned Dups,
+ BranchProbability Prediction, bool Forked) const {
+ const MachineFunction &MF = *TBBInfo.BB->getParent();
+ if (MF.getFunction().hasMinSize()) {
+ MachineBasicBlock::iterator TIB = TBBInfo.BB->begin();
+ MachineBasicBlock::iterator FIB = FBBInfo.BB->begin();
+ MachineBasicBlock::iterator TIE = TBBInfo.BB->end();
+ MachineBasicBlock::iterator FIE = FBBInfo.BB->end();
+
+ unsigned Dups1, Dups2;
+ if (!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2,
+ *TBBInfo.BB, *FBBInfo.BB,
+ /*SkipUnconditionalBranches*/ true))
+ llvm_unreachable("should already have been checked by ValidDiamond");
+
+ unsigned BranchBytes = 0;
+ unsigned CommonBytes = 0;
+
+ // Count common instructions at the start of the true and false blocks.
+ for (auto &I : make_range(TBBInfo.BB->begin(), TIB)) {
+ LLVM_DEBUG(dbgs() << "Common inst: " << I);
+ CommonBytes += TII->getInstSizeInBytes(I);
+ }
+ for (auto &I : make_range(FBBInfo.BB->begin(), FIB)) {
+ LLVM_DEBUG(dbgs() << "Common inst: " << I);
+ CommonBytes += TII->getInstSizeInBytes(I);
+ }
+
+ // Count instructions at the end of the true and false blocks, after
+ // the ones we plan to predicate. Analyzable branches will be removed
+ // (unless this is a forked diamond), and all other instructions are
+ // common between the two blocks.
+ for (auto &I : make_range(TIE, TBBInfo.BB->end())) {
+ if (I.isBranch() && TBBInfo.IsBrAnalyzable && !Forked) {
+ LLVM_DEBUG(dbgs() << "Saving branch: " << I);
+ BranchBytes += TII->predictBranchSizeForIfCvt(I);
+ } else {
+ LLVM_DEBUG(dbgs() << "Common inst: " << I);
+ CommonBytes += TII->getInstSizeInBytes(I);
+ }
+ }
+ for (auto &I : make_range(FIE, FBBInfo.BB->end())) {
+ if (I.isBranch() && FBBInfo.IsBrAnalyzable && !Forked) {
+ LLVM_DEBUG(dbgs() << "Saving branch: " << I);
+ BranchBytes += TII->predictBranchSizeForIfCvt(I);
+ } else {
+ LLVM_DEBUG(dbgs() << "Common inst: " << I);
+ CommonBytes += TII->getInstSizeInBytes(I);
+ }
+ }
+ for (auto &I : CommBB.terminators()) {
+ if (I.isBranch()) {
+ LLVM_DEBUG(dbgs() << "Saving branch: " << I);
+ BranchBytes += TII->predictBranchSizeForIfCvt(I);
+ }
+ }
+
+ // The common instructions in one branch will be eliminated, halving
+ // their code size.
+ CommonBytes /= 2;
+
+ // Count the instructions which we need to predicate.
+ unsigned NumPredicatedInstructions = 0;
+ for (auto &I : make_range(TIB, TIE)) {
+ if (!I.isDebugInstr()) {
+ LLVM_DEBUG(dbgs() << "Predicating: " << I);
+ NumPredicatedInstructions++;
+ }
+ }
+ for (auto &I : make_range(FIB, FIE)) {
+ if (!I.isDebugInstr()) {
+ LLVM_DEBUG(dbgs() << "Predicating: " << I);
+ NumPredicatedInstructions++;
+ }
+ }
+
+ // Even though we're optimising for size at the expense of performance,
+ // avoid creating really long predicated blocks.
+ if (NumPredicatedInstructions > 15)
+ return false;
+
+ // Some targets (e.g. Thumb2) need to insert extra instructions to
+ // start predicated blocks.
+ unsigned ExtraPredicateBytes = TII->extraSizeToPredicateInstructions(
+ MF, NumPredicatedInstructions);
+
+ LLVM_DEBUG(dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes
+ << ", CommonBytes=" << CommonBytes
+ << ", NumPredicatedInstructions="
+ << NumPredicatedInstructions
+ << ", ExtraPredicateBytes=" << ExtraPredicateBytes
+ << ")\n");
+ return (BranchBytes + CommonBytes) > ExtraPredicateBytes;
+ } else {
+ unsigned TCycle = TBBInfo.NonPredSize + TBBInfo.ExtraCost - Dups;
+ unsigned FCycle = FBBInfo.NonPredSize + FBBInfo.ExtraCost - Dups;
+ bool Res = TCycle > 0 && FCycle > 0 &&
+ TII->isProfitableToIfCvt(
+ *TBBInfo.BB, TCycle, TBBInfo.ExtraCost2, *FBBInfo.BB,
+ FCycle, FBBInfo.ExtraCost2, Prediction);
+ LLVM_DEBUG(dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycle
+ << ", FCycle=" << FCycle
+ << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra="
+ << FBBInfo.ExtraCost2 << ") = " << Res << "\n");
+ return Res;
+ }
}
/// Returns true if Block ends without a terminator.
@@ -333,6 +436,7 @@ char &llvm::IfConverterID = IfConverter::ID;
INITIALIZE_PASS_BEGIN(IfConverter, DEBUG_TYPE, "If Converter", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
+INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
INITIALIZE_PASS_END(IfConverter, DEBUG_TYPE, "If Converter", false, false)
bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
@@ -345,6 +449,8 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
TRI = ST.getRegisterInfo();
BranchFolder::MBFIWrapper MBFI(getAnalysis<MachineBlockFrequencyInfo>());
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
+ ProfileSummaryInfo *PSI =
+ &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
MRI = &MF.getRegInfo();
SchedModel.init(&ST);
@@ -355,9 +461,11 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
bool BFChange = false;
if (!PreRegAlloc) {
// Tail merge tend to expose more if-conversion opportunities.
- BranchFolder BF(true, false, MBFI, *MBPI);
- BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo(),
- getAnalysisIfAvailable<MachineModuleInfo>());
+ BranchFolder BF(true, false, MBFI, *MBPI, PSI);
+ auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+ BFChange = BF.OptimizeFunction(
+ MF, TII, ST.getRegisterInfo(),
+ MMIWP ? &MMIWP->getMMI() : nullptr);
}
LLVM_DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'"
@@ -495,9 +603,11 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
BBAnalysis.clear();
if (MadeChange && IfCvtBranchFold) {
- BranchFolder BF(false, false, MBFI, *MBPI);
- BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(),
- getAnalysisIfAvailable<MachineModuleInfo>());
+ BranchFolder BF(false, false, MBFI, *MBPI, PSI);
+ auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+ BF.OptimizeFunction(
+ MF, TII, MF.getSubtarget().getRegisterInfo(),
+ MMIWP ? &MMIWP->getMMI() : nullptr);
}
MadeChange |= BFChange;
@@ -569,6 +679,9 @@ bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
bool FalseBranch, unsigned &Dups,
BranchProbability Prediction) const {
Dups = 0;
+ if (TrueBBI.BB == FalseBBI.BB)
+ return false;
+
if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
return false;
@@ -835,6 +948,8 @@ bool IfConverter::ValidForkedDiamond(
TrueBBICalc.BB = TrueBBI.BB;
FalseBBICalc.BB = FalseBBI.BB;
+ TrueBBICalc.IsBrAnalyzable = TrueBBI.IsBrAnalyzable;
+ FalseBBICalc.IsBrAnalyzable = FalseBBI.IsBrAnalyzable;
if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc))
return false;
@@ -892,6 +1007,8 @@ bool IfConverter::ValidDiamond(
TrueBBICalc.BB = TrueBBI.BB;
FalseBBICalc.BB = FalseBBI.BB;
+ TrueBBICalc.IsBrAnalyzable = TrueBBI.IsBrAnalyzable;
+ FalseBBICalc.IsBrAnalyzable = FalseBBI.IsBrAnalyzable;
if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc))
return false;
// The size is used to decide whether to if-convert, and the shared portions
@@ -1179,13 +1296,9 @@ void IfConverter::AnalyzeBlock(
if (CanRevCond) {
BBInfo TrueBBICalc, FalseBBICalc;
- auto feasibleDiamond = [&]() {
- bool MeetsSize = MeetIfcvtSizeLimit(
- *TrueBBI.BB, (TrueBBICalc.NonPredSize - (Dups + Dups2) +
- TrueBBICalc.ExtraCost), TrueBBICalc.ExtraCost2,
- *FalseBBI.BB, (FalseBBICalc.NonPredSize - (Dups + Dups2) +
- FalseBBICalc.ExtraCost), FalseBBICalc.ExtraCost2,
- Prediction);
+ auto feasibleDiamond = [&](bool Forked) {
+ bool MeetsSize = MeetIfcvtSizeLimit(TrueBBICalc, FalseBBICalc, *BB,
+ Dups + Dups2, Prediction, Forked);
bool TrueFeasible = FeasibilityAnalysis(TrueBBI, BBI.BrCond,
/* IsTriangle */ false, /* RevCond */ false,
/* hasCommonTail */ true);
@@ -1197,7 +1310,7 @@ void IfConverter::AnalyzeBlock(
if (ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2,
TrueBBICalc, FalseBBICalc)) {
- if (feasibleDiamond()) {
+ if (feasibleDiamond(false)) {
// Diamond:
// EBB
// / \_
@@ -1206,14 +1319,14 @@ void IfConverter::AnalyzeBlock(
// \ /
// TailBB
// Note TailBB can be empty.
- Tokens.push_back(llvm::make_unique<IfcvtToken>(
+ Tokens.push_back(std::make_unique<IfcvtToken>(
BBI, ICDiamond, TNeedSub | FNeedSub, Dups, Dups2,
(bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred));
Enqueued = true;
}
} else if (ValidForkedDiamond(TrueBBI, FalseBBI, Dups, Dups2,
TrueBBICalc, FalseBBICalc)) {
- if (feasibleDiamond()) {
+ if (feasibleDiamond(true)) {
// ForkedDiamond:
// if TBB and FBB have a common tail that includes their conditional
// branch instructions, then we can If Convert this pattern.
@@ -1224,7 +1337,7 @@ void IfConverter::AnalyzeBlock(
// / \ / \
// FalseBB TrueBB FalseBB
//
- Tokens.push_back(llvm::make_unique<IfcvtToken>(
+ Tokens.push_back(std::make_unique<IfcvtToken>(
BBI, ICForkedDiamond, TNeedSub | FNeedSub, Dups, Dups2,
(bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred));
Enqueued = true;
@@ -1244,7 +1357,7 @@ void IfConverter::AnalyzeBlock(
// | /
// FBB
Tokens.push_back(
- llvm::make_unique<IfcvtToken>(BBI, ICTriangle, TNeedSub, Dups));
+ std::make_unique<IfcvtToken>(BBI, ICTriangle, TNeedSub, Dups));
Enqueued = true;
}
@@ -1253,7 +1366,7 @@ void IfConverter::AnalyzeBlock(
TrueBBI.ExtraCost2, Prediction) &&
FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) {
Tokens.push_back(
- llvm::make_unique<IfcvtToken>(BBI, ICTriangleRev, TNeedSub, Dups));
+ std::make_unique<IfcvtToken>(BBI, ICTriangleRev, TNeedSub, Dups));
Enqueued = true;
}
@@ -1269,7 +1382,7 @@ void IfConverter::AnalyzeBlock(
// |
// FBB
Tokens.push_back(
- llvm::make_unique<IfcvtToken>(BBI, ICSimple, TNeedSub, Dups));
+ std::make_unique<IfcvtToken>(BBI, ICSimple, TNeedSub, Dups));
Enqueued = true;
}
@@ -1281,7 +1394,7 @@ void IfConverter::AnalyzeBlock(
FalseBBI.NonPredSize + FalseBBI.ExtraCost,
FalseBBI.ExtraCost2, Prediction.getCompl()) &&
FeasibilityAnalysis(FalseBBI, RevCond, true)) {
- Tokens.push_back(llvm::make_unique<IfcvtToken>(BBI, ICTriangleFalse,
+ Tokens.push_back(std::make_unique<IfcvtToken>(BBI, ICTriangleFalse,
FNeedSub, Dups));
Enqueued = true;
}
@@ -1293,7 +1406,7 @@ void IfConverter::AnalyzeBlock(
FalseBBI.ExtraCost2, Prediction.getCompl()) &&
FeasibilityAnalysis(FalseBBI, RevCond, true, true)) {
Tokens.push_back(
- llvm::make_unique<IfcvtToken>(BBI, ICTriangleFRev, FNeedSub, Dups));
+ std::make_unique<IfcvtToken>(BBI, ICTriangleFRev, FNeedSub, Dups));
Enqueued = true;
}
@@ -1303,7 +1416,7 @@ void IfConverter::AnalyzeBlock(
FalseBBI.ExtraCost2, Prediction.getCompl()) &&
FeasibilityAnalysis(FalseBBI, RevCond)) {
Tokens.push_back(
- llvm::make_unique<IfcvtToken>(BBI, ICSimpleFalse, FNeedSub, Dups));
+ std::make_unique<IfcvtToken>(BBI, ICSimpleFalse, FNeedSub, Dups));
Enqueued = true;
}
}
@@ -1736,6 +1849,11 @@ bool IfConverter::IfConvertDiamondCommon(
++i;
}
while (NumDups1 != 0) {
+ // Since this instruction is going to be deleted, update call
+ // site info state if the instruction is call instruction.
+ if (DI2->isCall(MachineInstr::IgnoreBundle))
+ MBB2.getParent()->eraseCallSiteInfo(&*DI2);
+
++DI2;
if (DI2 == MBB2.end())
break;
@@ -1777,7 +1895,14 @@ bool IfConverter::IfConvertDiamondCommon(
// NumDups2 only counted non-dbg_value instructions, so this won't
// run off the head of the list.
assert(DI1 != MBB1.begin());
+
--DI1;
+
+ // Since this instruction is going to be deleted, update call
+ // site info state if the instruction is call instruction.
+ if (DI1->isCall(MachineInstr::IgnoreBundle))
+ MBB1.getParent()->eraseCallSiteInfo(&*DI1);
+
// skip dbg_value instructions
if (!DI1->isDebugInstr())
++i;
@@ -1827,7 +1952,7 @@ bool IfConverter::IfConvertDiamondCommon(
for (const MachineOperand &MO : FI.operands()) {
if (!MO.isReg())
continue;
- unsigned Reg = MO.getReg();
+ Register Reg = MO.getReg();
if (!Reg)
continue;
if (MO.isDef()) {
@@ -1995,7 +2120,7 @@ static bool MaySpeculate(const MachineInstr &MI,
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg())
continue;
- unsigned Reg = MO.getReg();
+ Register Reg = MO.getReg();
if (!Reg)
continue;
if (MO.isDef() && !LaterRedefs.count(Reg))
@@ -2062,6 +2187,10 @@ void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
break;
MachineInstr *MI = MF.CloneMachineInstr(&I);
+ // Make a copy of the call site info.
+ if (MI->isCall(MachineInstr::IgnoreBundle))
+ MF.copyCallSiteInfo(&I,MI);
+
ToBBI.BB->insert(ToBBI.BB->end(), MI);
ToBBI.NonPredSize++;
unsigned ExtraPredCost = TII->getPredicationCost(I);
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-03 10:30:24 +0000