summaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/MachineLICM.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/MachineLICM.cpp')
-rw-r--r--llvm/lib/CodeGen/MachineLICM.cpp169
1 files changed, 79 insertions, 90 deletions
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 5e8a916b3b3b..c06bc39b4940 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -42,6 +42,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCRegister.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
@@ -90,7 +91,7 @@ static cl::opt<UseBFI>
DisableHoistingToHotterBlocks("disable-hoisting-to-hotter-blocks",
cl::desc("Disable hoisting instructions to"
" hotter blocks"),
- cl::init(UseBFI::None), cl::Hidden,
+ cl::init(UseBFI::PGO), cl::Hidden,
cl::values(clEnumValN(UseBFI::None, "none",
"disable the feature"),
clEnumValN(UseBFI::PGO, "pgo",
@@ -145,7 +146,7 @@ namespace {
}
// Track 'estimated' register pressure.
- SmallSet<unsigned, 32> RegSeen;
+ SmallSet<Register, 32> RegSeen;
SmallVector<unsigned, 8> RegPressure;
// Register pressure "limit" per register pressure set. If the pressure
@@ -156,7 +157,7 @@ namespace {
SmallVector<SmallVector<unsigned, 8>, 16> BackTrace;
// For each opcode, keep a list of potential CSE instructions.
- DenseMap<unsigned, std::vector<const MachineInstr *>> CSEMap;
+ DenseMap<unsigned, std::vector<MachineInstr *>> CSEMap;
enum {
SpeculateFalse = 0,
@@ -212,7 +213,7 @@ namespace {
BitVector &PhysRegClobbers, SmallSet<int, 32> &StoredFIs,
SmallVectorImpl<CandidateInfo> &Candidates);
- void AddToLiveIns(unsigned Reg);
+ void AddToLiveIns(MCRegister Reg);
bool IsLICMCandidate(MachineInstr &I);
@@ -221,7 +222,7 @@ namespace {
bool HasLoopPHIUse(const MachineInstr *MI) const;
bool HasHighOperandLatency(MachineInstr &MI, unsigned DefIdx,
- unsigned Reg) const;
+ Register Reg) const;
bool IsCheapInstruction(MachineInstr &MI) const;
@@ -245,8 +246,6 @@ namespace {
void HoistOutOfLoop(MachineDomTreeNode *HeaderN);
- void HoistRegion(MachineDomTreeNode *N, bool IsHeader);
-
void SinkIntoLoop();
void InitRegPressure(MachineBasicBlock *BB);
@@ -260,13 +259,12 @@ namespace {
MachineInstr *ExtractHoistableLoad(MachineInstr *MI);
- const MachineInstr *
- LookForDuplicate(const MachineInstr *MI,
- std::vector<const MachineInstr *> &PrevMIs);
+ MachineInstr *LookForDuplicate(const MachineInstr *MI,
+ std::vector<MachineInstr *> &PrevMIs);
- bool EliminateCSE(
- MachineInstr *MI,
- DenseMap<unsigned, std::vector<const MachineInstr *>>::iterator &CI);
+ bool
+ EliminateCSE(MachineInstr *MI,
+ DenseMap<unsigned, std::vector<MachineInstr *>>::iterator &CI);
bool MayCSE(MachineInstr *MI);
@@ -606,7 +604,7 @@ void MachineLICMBase::HoistRegionPostRA() {
/// Add register 'Reg' to the livein sets of BBs in the current loop, and make
/// sure it is not killed by any instructions in the loop.
-void MachineLICMBase::AddToLiveIns(unsigned Reg) {
+void MachineLICMBase::AddToLiveIns(MCRegister Reg) {
for (MachineBasicBlock *BB : CurLoop->getBlocks()) {
if (!BB->isLiveIn(Reg))
BB->addLiveIn(Reg);
@@ -802,8 +800,13 @@ void MachineLICMBase::SinkIntoLoop() {
I != Preheader->instr_end(); ++I) {
// We need to ensure that we can safely move this instruction into the loop.
// As such, it must not have side-effects, e.g. such as a call has.
- if (IsLoopInvariantInst(*I) && !HasLoopPHIUse(&*I))
+ LLVM_DEBUG(dbgs() << "LICM: Analysing sink candidate: " << *I);
+ if (IsLoopInvariantInst(*I) && !HasLoopPHIUse(&*I)) {
+ LLVM_DEBUG(dbgs() << "LICM: Added as sink candidate.\n");
Candidates.push_back(&*I);
+ continue;
+ }
+ LLVM_DEBUG(dbgs() << "LICM: Not added as sink candidate.\n");
}
for (MachineInstr *I : Candidates) {
@@ -813,8 +816,11 @@ void MachineLICMBase::SinkIntoLoop() {
if (!MRI->hasOneDef(MO.getReg()))
continue;
bool CanSink = true;
- MachineBasicBlock *B = nullptr;
+ MachineBasicBlock *SinkBlock = nullptr;
+ LLVM_DEBUG(dbgs() << "LICM: Try sinking: " << *I);
+
for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) {
+ LLVM_DEBUG(dbgs() << "LICM: Analysing use: "; MI.dump());
// FIXME: Come up with a proper cost model that estimates whether sinking
// the instruction (and thus possibly executing it on every loop
// iteration) is more expensive than a register.
@@ -823,24 +829,40 @@ void MachineLICMBase::SinkIntoLoop() {
CanSink = false;
break;
}
- if (!B) {
- B = MI.getParent();
+ if (!SinkBlock) {
+ SinkBlock = MI.getParent();
+ LLVM_DEBUG(dbgs() << "LICM: Setting sink block to: "
+ << printMBBReference(*SinkBlock) << "\n");
continue;
}
- B = DT->findNearestCommonDominator(B, MI.getParent());
- if (!B) {
+ SinkBlock = DT->findNearestCommonDominator(SinkBlock, MI.getParent());
+ if (!SinkBlock) {
+ LLVM_DEBUG(dbgs() << "LICM: Can't find nearest dominator\n");
CanSink = false;
break;
}
+ LLVM_DEBUG(dbgs() << "LICM: Setting nearest common dom block: " <<
+ printMBBReference(*SinkBlock) << "\n");
+ }
+ if (!CanSink) {
+ LLVM_DEBUG(dbgs() << "LICM: Can't sink instruction.\n");
+ continue;
+ }
+ if (!SinkBlock) {
+ LLVM_DEBUG(dbgs() << "LICM: Not sinking, can't find sink block.\n");
+ continue;
}
- if (!CanSink || !B || B == Preheader)
+ if (SinkBlock == Preheader) {
+ LLVM_DEBUG(dbgs() << "LICM: Not sinking, sink block is the preheader\n");
continue;
+ }
- LLVM_DEBUG(dbgs() << "Sinking to " << printMBBReference(*B) << " from "
- << printMBBReference(*I->getParent()) << ": " << *I);
- B->splice(B->getFirstNonPHI(), Preheader, I);
+ LLVM_DEBUG(dbgs() << "LICM: Sinking to " << printMBBReference(*SinkBlock)
+ << " from " << printMBBReference(*I->getParent())
+ << ": " << *I);
+ SinkBlock->splice(SinkBlock->getFirstNonPHI(), Preheader, I);
- // The instruction is is moved from its basic block, so do not retain the
+ // The instruction is moved from its basic block, so do not retain the
// debug information.
assert(!I->isDebugInstr() && "Should not sink debug inst");
I->setDebugLoc(DebugLoc());
@@ -978,7 +1000,7 @@ static bool isInvariantStore(const MachineInstr &MI,
Reg = TRI->lookThruCopyLike(MO.getReg(), MRI);
if (Register::isVirtualRegister(Reg))
return false;
- if (!TRI->isCallerPreservedPhysReg(Reg, *MI.getMF()))
+ if (!TRI->isCallerPreservedPhysReg(Reg.asMCReg(), *MI.getMF()))
return false;
else
FoundCallerPresReg = true;
@@ -1008,7 +1030,7 @@ static bool isCopyFeedingInvariantStore(const MachineInstr &MI,
if (Register::isVirtualRegister(CopySrcReg))
return false;
- if (!TRI->isCallerPreservedPhysReg(CopySrcReg, *MF))
+ if (!TRI->isCallerPreservedPhysReg(CopySrcReg.asMCReg(), *MF))
return false;
Register CopyDstReg = MI.getOperand(0).getReg();
@@ -1030,6 +1052,7 @@ bool MachineLICMBase::IsLICMCandidate(MachineInstr &I) {
bool DontMoveAcrossStore = true;
if ((!I.isSafeToMove(AA, DontMoveAcrossStore)) &&
!(HoistConstStores && isInvariantStore(I, TRI, MRI))) {
+ LLVM_DEBUG(dbgs() << "LICM: Instruction not safe to move.\n");
return false;
}
@@ -1040,65 +1063,28 @@ bool MachineLICMBase::IsLICMCandidate(MachineInstr &I) {
// indexed load from a jump table.
// Stores and side effects are already checked by isSafeToMove.
if (I.mayLoad() && !mayLoadFromGOTOrConstantPool(I) &&
- !IsGuaranteedToExecute(I.getParent()))
+ !IsGuaranteedToExecute(I.getParent())) {
+ LLVM_DEBUG(dbgs() << "LICM: Load not guaranteed to execute.\n");
+ return false;
+ }
+
+ // Convergent attribute has been used on operations that involve inter-thread
+ // communication which results are implicitly affected by the enclosing
+ // control flows. It is not safe to hoist or sink such operations across
+ // control flow.
+ if (I.isConvergent())
return false;
return true;
}
/// Returns true if the instruction is loop invariant.
-/// I.e., all virtual register operands are defined outside of the loop,
-/// physical registers aren't accessed explicitly, and there are no side
-/// effects that aren't captured by the operands or other flags.
bool MachineLICMBase::IsLoopInvariantInst(MachineInstr &I) {
- if (!IsLICMCandidate(I))
+ if (!IsLICMCandidate(I)) {
+ LLVM_DEBUG(dbgs() << "LICM: Instruction not a LICM candidate\n");
return false;
-
- // The instruction is loop invariant if all of its operands are.
- for (const MachineOperand &MO : I.operands()) {
- if (!MO.isReg())
- continue;
-
- Register Reg = MO.getReg();
- if (Reg == 0) continue;
-
- // Don't hoist an instruction that uses or defines a physical register.
- if (Register::isPhysicalRegister(Reg)) {
- if (MO.isUse()) {
- // If the physreg has no defs anywhere, it's just an ambient register
- // and we can freely move its uses. Alternatively, if it's allocatable,
- // it could get allocated to something with a def during allocation.
- // However, if the physreg is known to always be caller saved/restored
- // then this use is safe to hoist.
- if (!MRI->isConstantPhysReg(Reg) &&
- !(TRI->isCallerPreservedPhysReg(Reg, *I.getMF())))
- return false;
- // Otherwise it's safe to move.
- continue;
- } else if (!MO.isDead()) {
- // A def that isn't dead. We can't move it.
- return false;
- } else if (CurLoop->getHeader()->isLiveIn(Reg)) {
- // If the reg is live into the loop, we can't hoist an instruction
- // which would clobber it.
- return false;
- }
- }
-
- if (!MO.isUse())
- continue;
-
- assert(MRI->getVRegDef(Reg) &&
- "Machine instr not mapped for this vreg?!");
-
- // If the loop contains the definition of an operand, then the instruction
- // isn't loop invariant.
- if (CurLoop->contains(MRI->getVRegDef(Reg)))
- return false;
}
-
- // If we got this far, the instruction is loop invariant!
- return true;
+ return CurLoop->isLoopInvariant(I);
}
/// Return true if the specified instruction is used by a phi node and hoisting
@@ -1138,9 +1124,8 @@ bool MachineLICMBase::HasLoopPHIUse(const MachineInstr *MI) const {
/// Compute operand latency between a def of 'Reg' and an use in the current
/// loop, return true if the target considered it high.
-bool MachineLICMBase::HasHighOperandLatency(MachineInstr &MI,
- unsigned DefIdx,
- unsigned Reg) const {
+bool MachineLICMBase::HasHighOperandLatency(MachineInstr &MI, unsigned DefIdx,
+ Register Reg) const {
if (MRI->use_nodbg_empty(Reg))
return false;
@@ -1400,10 +1385,10 @@ void MachineLICMBase::InitCSEMap(MachineBasicBlock *BB) {
/// Find an instruction amount PrevMIs that is a duplicate of MI.
/// Return this instruction if it's found.
-const MachineInstr*
+MachineInstr *
MachineLICMBase::LookForDuplicate(const MachineInstr *MI,
- std::vector<const MachineInstr*> &PrevMIs) {
- for (const MachineInstr *PrevMI : PrevMIs)
+ std::vector<MachineInstr *> &PrevMIs) {
+ for (MachineInstr *PrevMI : PrevMIs)
if (TII->produceSameValue(*MI, *PrevMI, (PreRegAlloc ? MRI : nullptr)))
return PrevMI;
@@ -1414,14 +1399,15 @@ MachineLICMBase::LookForDuplicate(const MachineInstr *MI,
/// computes the same value. If it's found, do a RAU on with the definition of
/// the existing instruction rather than hoisting the instruction to the
/// preheader.
-bool MachineLICMBase::EliminateCSE(MachineInstr *MI,
- DenseMap<unsigned, std::vector<const MachineInstr *>>::iterator &CI) {
+bool MachineLICMBase::EliminateCSE(
+ MachineInstr *MI,
+ DenseMap<unsigned, std::vector<MachineInstr *>>::iterator &CI) {
// Do not CSE implicit_def so ProcessImplicitDefs can properly propagate
// the undef property onto uses.
if (CI == CSEMap.end() || MI->isImplicitDef())
return false;
- if (const MachineInstr *Dup = LookForDuplicate(MI, CI->second)) {
+ if (MachineInstr *Dup = LookForDuplicate(MI, CI->second)) {
LLVM_DEBUG(dbgs() << "CSEing " << *MI << " with " << *Dup);
// Replace virtual registers defined by MI by their counterparts defined
@@ -1461,6 +1447,9 @@ bool MachineLICMBase::EliminateCSE(MachineInstr *MI,
Register DupReg = Dup->getOperand(Idx).getReg();
MRI->replaceRegWith(Reg, DupReg);
MRI->clearKillFlags(DupReg);
+ // Clear Dup dead flag if any, we reuse it for Reg.
+ if (!MRI->use_nodbg_empty(DupReg))
+ Dup->getOperand(Idx).setIsDead(false);
}
MI->eraseFromParent();
@@ -1474,8 +1463,8 @@ bool MachineLICMBase::EliminateCSE(MachineInstr *MI,
/// the loop.
bool MachineLICMBase::MayCSE(MachineInstr *MI) {
unsigned Opcode = MI->getOpcode();
- DenseMap<unsigned, std::vector<const MachineInstr *>>::iterator
- CI = CSEMap.find(Opcode);
+ DenseMap<unsigned, std::vector<MachineInstr *>>::iterator CI =
+ CSEMap.find(Opcode);
// Do not CSE implicit_def so ProcessImplicitDefs can properly propagate
// the undef property onto uses.
if (CI == CSEMap.end() || MI->isImplicitDef())
@@ -1529,8 +1518,8 @@ bool MachineLICMBase::Hoist(MachineInstr *MI, MachineBasicBlock *Preheader) {
// Look for opportunity to CSE the hoisted instruction.
unsigned Opcode = MI->getOpcode();
- DenseMap<unsigned, std::vector<const MachineInstr *>>::iterator
- CI = CSEMap.find(Opcode);
+ DenseMap<unsigned, std::vector<MachineInstr *>>::iterator CI =
+ CSEMap.find(Opcode);
if (!EliminateCSE(MI, CI)) {
// Otherwise, splice the instruction to the preheader.
Preheader->splice(Preheader->getFirstTerminator(),MI->getParent(),MI);
generated by cgit v1.3 (git 2.53.0) at 2026-05-12 08:19:23 +0000