aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/BranchFolding.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/BranchFolding.cpp')
-rw-r--r--llvm/lib/CodeGen/BranchFolding.cpp172
1 files changed, 65 insertions, 107 deletions
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 4b9c50aeb1d3..c6d5aa37834f 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -40,6 +40,7 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineSizeOpts.h"
+#include "llvm/CodeGen/MBFIWrapper.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -129,15 +130,13 @@ bool BranchFolderPass::runOnMachineFunction(MachineFunction &MF) {
// HW that requires structurized CFG.
bool EnableTailMerge = !MF.getTarget().requiresStructuredCFG() &&
PassConfig->getEnableTailMerge();
- BranchFolder::MBFIWrapper MBBFreqInfo(
+ MBFIWrapper MBBFreqInfo(
getAnalysis<MachineBlockFrequencyInfo>());
BranchFolder Folder(EnableTailMerge, /*CommonHoist=*/true, MBBFreqInfo,
getAnalysis<MachineBranchProbabilityInfo>(),
&getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI());
- auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
- return Folder.OptimizeFunction(
- MF, MF.getSubtarget().getInstrInfo(), MF.getSubtarget().getRegisterInfo(),
- MMIWP ? &MMIWP->getMMI() : nullptr);
+ return Folder.OptimizeFunction(MF, MF.getSubtarget().getInstrInfo(),
+ MF.getSubtarget().getRegisterInfo());
}
BranchFolder::BranchFolder(bool defaultEnableTailMerge, bool CommonHoist,
@@ -170,7 +169,7 @@ void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
// Update call site info.
std::for_each(MBB->begin(), MBB->end(), [MF](const MachineInstr &MI) {
- if (MI.isCall(MachineInstr::IgnoreBundle))
+ if (MI.shouldUpdateCallSiteInfo())
MF->eraseCallSiteInfo(&MI);
});
// Remove the block.
@@ -183,7 +182,6 @@ void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
bool BranchFolder::OptimizeFunction(MachineFunction &MF,
const TargetInstrInfo *tii,
const TargetRegisterInfo *tri,
- MachineModuleInfo *mmi,
MachineLoopInfo *mli, bool AfterPlacement) {
if (!tii) return false;
@@ -193,7 +191,6 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF,
AfterBlockPlacement = AfterPlacement;
TII = tii;
TRI = tri;
- MMI = mmi;
MLI = mli;
this->MRI = &MRI;
@@ -201,14 +198,7 @@ bool BranchFolder::OptimizeFunction(MachineFunction &MF,
if (!UpdateLiveIns)
MRI.invalidateLiveness();
- // Fix CFG. The later algorithms expect it to be right.
bool MadeChange = false;
- for (MachineBasicBlock &MBB : MF) {
- MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
- SmallVector<MachineOperand, 4> Cond;
- if (!TII->analyzeBranch(MBB, TBB, FBB, Cond, true))
- MadeChange |= MBB.CorrectExtraCFGEdges(TBB, FBB, !Cond.empty());
- }
// Recalculate EH scope membership.
EHScopeMembership = getEHScopeMembership(MF);
@@ -354,6 +344,9 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
MBBI1->isInlineAsm()) {
break;
}
+ if (MBBI1->getFlag(MachineInstr::NoMerge) ||
+ MBBI2->getFlag(MachineInstr::NoMerge))
+ break;
++TailLen;
I1 = MBBI1;
I2 = MBBI2;
@@ -501,42 +494,6 @@ BranchFolder::MergePotentialsElt::operator<(const MergePotentialsElt &o) const {
#endif
}
-BlockFrequency
-BranchFolder::MBFIWrapper::getBlockFreq(const MachineBasicBlock *MBB) const {
- auto I = MergedBBFreq.find(MBB);
-
- if (I != MergedBBFreq.end())
- return I->second;
-
- return MBFI.getBlockFreq(MBB);
-}
-
-void BranchFolder::MBFIWrapper::setBlockFreq(const MachineBasicBlock *MBB,
- BlockFrequency F) {
- MergedBBFreq[MBB] = F;
-}
-
-raw_ostream &
-BranchFolder::MBFIWrapper::printBlockFreq(raw_ostream &OS,
- const MachineBasicBlock *MBB) const {
- return MBFI.printBlockFreq(OS, getBlockFreq(MBB));
-}
-
-raw_ostream &
-BranchFolder::MBFIWrapper::printBlockFreq(raw_ostream &OS,
- const BlockFrequency Freq) const {
- return MBFI.printBlockFreq(OS, Freq);
-}
-
-void BranchFolder::MBFIWrapper::view(const Twine &Name, bool isSimple) {
- MBFI.view(Name, isSimple);
-}
-
-uint64_t
-BranchFolder::MBFIWrapper::getEntryFreq() const {
- return MBFI.getEntryFreq();
-}
-
/// CountTerminators - Count the number of terminators in the given
/// block and set I to the position of the first non-terminator, if there
/// is one, or MBB->end() otherwise.
@@ -591,7 +548,7 @@ ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2,
MachineBasicBlock *PredBB,
DenseMap<const MachineBasicBlock *, int> &EHScopeMembership,
bool AfterPlacement,
- BranchFolder::MBFIWrapper &MBBFreqInfo,
+ MBFIWrapper &MBBFreqInfo,
ProfileSummaryInfo *PSI) {
// It is never profitable to tail-merge blocks from two different EH scopes.
if (!EHScopeMembership.empty()) {
@@ -691,8 +648,8 @@ ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2,
MachineFunction *MF = MBB1->getParent();
bool OptForSize =
MF->getFunction().hasOptSize() ||
- (llvm::shouldOptimizeForSize(MBB1, PSI, &MBBFreqInfo.getMBFI()) &&
- llvm::shouldOptimizeForSize(MBB2, PSI, &MBBFreqInfo.getMBFI()));
+ (llvm::shouldOptimizeForSize(MBB1, PSI, &MBBFreqInfo) &&
+ llvm::shouldOptimizeForSize(MBB2, PSI, &MBBFreqInfo));
return EffectiveTailLen >= 2 && OptForSize &&
(FullBlockTail1 || FullBlockTail2);
}
@@ -900,7 +857,7 @@ void BranchFolder::mergeCommonTails(unsigned commonTailIndex) {
LiveRegs.clear();
LiveRegs.addLiveOuts(*Pred);
MachineBasicBlock::iterator InsertBefore = Pred->getFirstTerminator();
- for (unsigned Reg : NewLiveIns) {
+ for (Register Reg : NewLiveIns) {
if (!LiveRegs.available(*MRI, Reg))
continue;
DebugLoc DL;
@@ -963,10 +920,10 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
continue;
}
- // If one of the blocks is the entire common tail (and not the entry
- // block, which we can't jump to), we can treat all blocks with this same
- // tail at once. Use PredBB if that is one of the possibilities, as that
- // will not introduce any extra branches.
+ // If one of the blocks is the entire common tail (and is not the entry
+ // block/an EH pad, which we can't jump to), we can treat all blocks with
+ // this same tail at once. Use PredBB if that is one of the possibilities,
+ // as that will not introduce any extra branches.
MachineBasicBlock *EntryBB =
&MergePotentials.front().getBlock()->getParent()->front();
unsigned commonTailIndex = SameTails.size();
@@ -974,19 +931,21 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
// into the other.
if (SameTails.size() == 2 &&
SameTails[0].getBlock()->isLayoutSuccessor(SameTails[1].getBlock()) &&
- SameTails[1].tailIsWholeBlock())
+ SameTails[1].tailIsWholeBlock() && !SameTails[1].getBlock()->isEHPad())
commonTailIndex = 1;
else if (SameTails.size() == 2 &&
SameTails[1].getBlock()->isLayoutSuccessor(
- SameTails[0].getBlock()) &&
- SameTails[0].tailIsWholeBlock())
+ SameTails[0].getBlock()) &&
+ SameTails[0].tailIsWholeBlock() &&
+ !SameTails[0].getBlock()->isEHPad())
commonTailIndex = 0;
else {
// Otherwise just pick one, favoring the fall-through predecessor if
// there is one.
for (unsigned i = 0, e = SameTails.size(); i != e; ++i) {
MachineBasicBlock *MBB = SameTails[i].getBlock();
- if (MBB == EntryBB && SameTails[i].tailIsWholeBlock())
+ if ((MBB == EntryBB || MBB->isEHPad()) &&
+ SameTails[i].tailIsWholeBlock())
continue;
if (MBB == PredBB) {
commonTailIndex = i;
@@ -1124,8 +1083,9 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
if (!UniquePreds.insert(PBB).second)
continue;
- // Skip blocks which may jump to a landing pad. Can't tail merge these.
- if (PBB->hasEHPadSuccessor())
+ // Skip blocks which may jump to a landing pad or jump from an asm blob.
+ // Can't tail merge these.
+ if (PBB->hasEHPadSuccessor() || PBB->mayHaveInlineAsmBr())
continue;
// After block placement, only consider predecessors that belong to the
@@ -1371,6 +1331,13 @@ ReoptimizeBlock:
SameEHScope = MBBEHScope->second == FallThroughEHScope->second;
}
+ // Analyze the branch in the current block. As a side-effect, this may cause
+ // the block to become empty.
+ MachineBasicBlock *CurTBB = nullptr, *CurFBB = nullptr;
+ SmallVector<MachineOperand, 4> CurCond;
+ bool CurUnAnalyzable =
+ TII->analyzeBranch(*MBB, CurTBB, CurFBB, CurCond, true);
+
// If this block is empty, make everyone use its fall-through, not the block
// explicitly. Landing pads should not do this since the landing-pad table
// points to this block. Blocks with their addresses taken shouldn't be
@@ -1413,10 +1380,6 @@ ReoptimizeBlock:
bool PriorUnAnalyzable =
TII->analyzeBranch(PrevBB, PriorTBB, PriorFBB, PriorCond, true);
if (!PriorUnAnalyzable) {
- // If the CFG for the prior block has extra edges, remove them.
- MadeChange |= PrevBB.CorrectExtraCFGEdges(PriorTBB, PriorFBB,
- !PriorCond.empty());
-
// If the previous branch is conditional and both conditions go to the same
// destination, remove the branch, replacing it with an unconditional one or
// a fall-through.
@@ -1437,7 +1400,7 @@ ReoptimizeBlock:
// has been used, but it can happen if tail merging splits a fall-through
// predecessor of a block.
// This has to check PrevBB->succ_size() because EH edges are ignored by
- // AnalyzeBranch.
+ // analyzeBranch.
if (PriorCond.empty() && !PriorTBB && MBB->pred_size() == 1 &&
PrevBB.succ_size() == 1 &&
!MBB->hasAddressTaken() && !MBB->isEHPad()) {
@@ -1547,7 +1510,7 @@ ReoptimizeBlock:
bool OptForSize =
MF.getFunction().hasOptSize() ||
- llvm::shouldOptimizeForSize(MBB, PSI, &MBBFreqInfo.getMBFI());
+ llvm::shouldOptimizeForSize(MBB, PSI, &MBBFreqInfo);
if (!IsEmptyBlock(MBB) && MBB->pred_size() == 1 && OptForSize) {
// Changing "Jcc foo; foo: jmp bar;" into "Jcc bar;" might change the branch
// direction, thereby defeating careful block placement and regressing
@@ -1584,15 +1547,7 @@ ReoptimizeBlock:
}
}
- // Analyze the branch in the current block.
- MachineBasicBlock *CurTBB = nullptr, *CurFBB = nullptr;
- SmallVector<MachineOperand, 4> CurCond;
- bool CurUnAnalyzable =
- TII->analyzeBranch(*MBB, CurTBB, CurFBB, CurCond, true);
if (!CurUnAnalyzable) {
- // If the CFG for the prior block has extra edges, remove them.
- MadeChange |= MBB->CorrectExtraCFGEdges(CurTBB, CurFBB, !CurCond.empty());
-
// If this is a two-way branch, and the FBB branches to this block, reverse
// the condition so the single-basic-block loop is faster. Instead of:
// Loop: xxx; jcc Out; jmp Loop
@@ -1669,7 +1624,7 @@ ReoptimizeBlock:
PMBB->ReplaceUsesOfBlockWith(MBB, CurTBB);
// If this change resulted in PMBB ending in a conditional
// branch where both conditions go to the same destination,
- // change this to an unconditional branch (and fix the CFG).
+ // change this to an unconditional branch.
MachineBasicBlock *NewCurTBB = nullptr, *NewCurFBB = nullptr;
SmallVector<MachineOperand, 4> NewCurCond;
bool NewCurUnAnalyzable = TII->analyzeBranch(
@@ -1681,7 +1636,6 @@ ReoptimizeBlock:
TII->insertBranch(*PMBB, NewCurTBB, nullptr, NewCurCond, pdl);
MadeChange = true;
++NumBranchOpts;
- PMBB->CorrectExtraCFGEdges(NewCurTBB, nullptr, false);
}
}
}
@@ -1712,13 +1666,15 @@ ReoptimizeBlock:
if (!MBB->isEHPad()) {
// Check all the predecessors of this block. If one of them has no fall
- // throughs, move this block right after it.
+ // throughs, and analyzeBranch thinks it _could_ fallthrough to this
+ // block, move this block right after it.
for (MachineBasicBlock *PredBB : MBB->predecessors()) {
// Analyze the branch at the end of the pred.
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (PredBB != MBB && !PredBB->canFallThrough() &&
!TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true) &&
+ (PredTBB == MBB || PredFBB == MBB) &&
(!CurFallsThru || !CurTBB || !CurFBB) &&
(!CurFallsThru || MBB->getNumber() >= PredBB->getNumber())) {
// If the current block doesn't fall through, just move it.
@@ -1744,21 +1700,24 @@ ReoptimizeBlock:
}
if (!CurFallsThru) {
- // Check all successors to see if we can move this block before it.
- for (MachineBasicBlock *SuccBB : MBB->successors()) {
- // Analyze the branch at the end of the block before the succ.
- MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
-
- // If this block doesn't already fall-through to that successor, and if
- // the succ doesn't already have a block that can fall through into it,
- // and if the successor isn't an EH destination, we can arrange for the
- // fallthrough to happen.
- if (SuccBB != MBB && &*SuccPrev != MBB &&
- !SuccPrev->canFallThrough() && !CurUnAnalyzable &&
- !SuccBB->isEHPad()) {
- MBB->moveBefore(SuccBB);
- MadeChange = true;
- goto ReoptimizeBlock;
+ // Check analyzable branch-successors to see if we can move this block
+ // before one.
+ if (!CurUnAnalyzable) {
+ for (MachineBasicBlock *SuccBB : {CurFBB, CurTBB}) {
+ if (!SuccBB)
+ continue;
+ // Analyze the branch at the end of the block before the succ.
+ MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
+
+ // If this block doesn't already fall-through to that successor, and
+ // if the succ doesn't already have a block that can fall through into
+ // it, we can arrange for the fallthrough to happen.
+ if (SuccBB != MBB && &*SuccPrev != MBB &&
+ !SuccPrev->canFallThrough()) {
+ MBB->moveBefore(SuccBB);
+ MadeChange = true;
+ goto ReoptimizeBlock;
+ }
}
}
@@ -1817,9 +1776,9 @@ static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB,
}
template <class Container>
-static void addRegAndItsAliases(unsigned Reg, const TargetRegisterInfo *TRI,
+static void addRegAndItsAliases(Register Reg, const TargetRegisterInfo *TRI,
Container &Set) {
- if (Register::isPhysicalRegister(Reg)) {
+ if (Reg.isPhysical()) {
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
Set.insert(*AI);
} else {
@@ -1838,8 +1797,8 @@ static
MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI,
- SmallSet<unsigned,4> &Uses,
- SmallSet<unsigned,4> &Defs) {
+ SmallSet<Register, 4> &Uses,
+ SmallSet<Register, 4> &Defs) {
MachineBasicBlock::iterator Loc = MBB->getFirstTerminator();
if (!TII->isUnpredicatedTerminator(*Loc))
return MBB->end();
@@ -1875,8 +1834,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
// The terminator is probably a conditional branch, try not to separate the
// branch from condition setting instruction.
- MachineBasicBlock::iterator PI =
- skipDebugInstructionsBackward(std::prev(Loc), MBB->begin());
+ MachineBasicBlock::iterator PI = prev_nodbg(Loc, MBB->begin());
bool IsDef = false;
for (const MachineOperand &MO : PI->operands()) {
@@ -1951,14 +1909,14 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
// Find a suitable position to hoist the common instructions to. Also figure
// out which registers are used or defined by instructions from the insertion
// point to the end of the block.
- SmallSet<unsigned, 4> Uses, Defs;
+ SmallSet<Register, 4> Uses, Defs;
MachineBasicBlock::iterator Loc =
findHoistingInsertPosAndDeps(MBB, TII, TRI, Uses, Defs);
if (Loc == MBB->end())
return false;
bool HasDups = false;
- SmallSet<unsigned, 4> ActiveDefsSet, AllDefsSet;
+ SmallSet<Register, 4> ActiveDefsSet, AllDefsSet;
MachineBasicBlock::iterator TIB = TBB->begin();
MachineBasicBlock::iterator FIB = FBB->begin();
MachineBasicBlock::iterator TIE = TBB->end();
@@ -2042,7 +2000,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
if (!AllDefsSet.count(Reg)) {
continue;
}
- if (Register::isPhysicalRegister(Reg)) {
+ if (Reg.isPhysical()) {
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
ActiveDefsSet.erase(*AI);
} else {
@@ -2055,7 +2013,7 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
if (!MO.isReg() || !MO.isDef() || MO.isDead())
continue;
Register Reg = MO.getReg();
- if (!Reg || Register::isVirtualRegister(Reg))
+ if (!Reg || Reg.isVirtual())
continue;
addRegAndItsAliases(Reg, TRI, ActiveDefsSet);
addRegAndItsAliases(Reg, TRI, AllDefsSet);
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-04 14:07:06 +0000