summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp279
1 files changed, 231 insertions, 48 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
index 41db2c88ce50..a4ba197b7a1d 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineSink.cpp
@@ -15,6 +15,8 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
@@ -36,8 +38,10 @@
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/CommandLine.h"
@@ -104,6 +108,25 @@ namespace {
using AllSuccsCache =
std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
+ /// DBG_VALUE pointer and flag. The flag is true if this DBG_VALUE is
+ /// post-dominated by another DBG_VALUE of the same variable location.
+ /// This is necessary to detect sequences such as:
+ /// %0 = someinst
+ /// DBG_VALUE %0, !123, !DIExpression()
+ /// %1 = anotherinst
+ /// DBG_VALUE %1, !123, !DIExpression()
+ /// Where if %0 were to sink, the DBG_VAUE should not sink with it, as that
+ /// would re-order assignments.
+ using SeenDbgUser = PointerIntPair<MachineInstr *, 1>;
+
+ /// Record of DBG_VALUE uses of vregs in a block, so that we can identify
+ /// debug instructions to sink.
+ SmallDenseMap<unsigned, TinyPtrVector<SeenDbgUser>> SeenDbgUsers;
+
+ /// Record of debug variables that have had their locations set in the
+ /// current block.
+ DenseSet<DebugVariable> SeenDbgVars;
+
public:
static char ID; // Pass identification
@@ -114,15 +137,12 @@ namespace {
bool runOnMachineFunction(MachineFunction &MF) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<MachineDominatorTree>();
AU.addRequired<MachinePostDominatorTree>();
AU.addRequired<MachineLoopInfo>();
AU.addRequired<MachineBranchProbabilityInfo>();
- AU.addPreserved<MachineDominatorTree>();
- AU.addPreserved<MachinePostDominatorTree>();
AU.addPreserved<MachineLoopInfo>();
if (UseBlockFreqInfo)
AU.addRequired<MachineBlockFrequencyInfo>();
@@ -134,6 +154,7 @@ namespace {
private:
bool ProcessBlock(MachineBasicBlock &MBB);
+ void ProcessDbgInst(MachineInstr &MI);
bool isWorthBreakingCriticalEdge(MachineInstr &MI,
MachineBasicBlock *From,
MachineBasicBlock *To);
@@ -155,8 +176,14 @@ namespace {
MachineBasicBlock *To,
bool BreakPHIEdge);
bool SinkInstruction(MachineInstr &MI, bool &SawStore,
-
AllSuccsCache &AllSuccessors);
+
+ /// If we sink a COPY inst, some debug users of it's destination may no
+ /// longer be dominated by the COPY, and will eventually be dropped.
+ /// This is easily rectified by forwarding the non-dominated debug uses
+ /// to the copy source.
+ void SalvageUnsunkDebugUsersOfCopy(MachineInstr &,
+ MachineBasicBlock *TargetBlock);
bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB,
MachineBasicBlock *DefMBB,
bool &BreakPHIEdge, bool &LocalUse) const;
@@ -195,11 +222,10 @@ bool MachineSinking::PerformTrivialForwardCoalescing(MachineInstr &MI,
if (!MI.isCopy())
return false;
- unsigned SrcReg = MI.getOperand(1).getReg();
- unsigned DstReg = MI.getOperand(0).getReg();
- if (!TargetRegisterInfo::isVirtualRegister(SrcReg) ||
- !TargetRegisterInfo::isVirtualRegister(DstReg) ||
- !MRI->hasOneNonDBGUse(SrcReg))
+ Register SrcReg = MI.getOperand(1).getReg();
+ Register DstReg = MI.getOperand(0).getReg();
+ if (!Register::isVirtualRegister(SrcReg) ||
+ !Register::isVirtualRegister(DstReg) || !MRI->hasOneNonDBGUse(SrcReg))
return false;
const TargetRegisterClass *SRC = MRI->getRegClass(SrcReg);
@@ -233,8 +259,7 @@ MachineSinking::AllUsesDominatedByBlock(unsigned Reg,
MachineBasicBlock *DefMBB,
bool &BreakPHIEdge,
bool &LocalUse) const {
- assert(TargetRegisterInfo::isVirtualRegister(Reg) &&
- "Only makes sense for vregs");
+ assert(Register::isVirtualRegister(Reg) && "Only makes sense for vregs");
// Ignore debug uses because debug info doesn't affect the code.
if (MRI->use_nodbg_empty(Reg))
@@ -371,8 +396,11 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
if (!ProcessedBegin)
--I;
- if (MI.isDebugInstr())
+ if (MI.isDebugInstr()) {
+ if (MI.isDebugValue())
+ ProcessDbgInst(MI);
continue;
+ }
bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
if (Joined) {
@@ -388,9 +416,29 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
// If we just processed the first instruction in the block, we're done.
} while (!ProcessedBegin);
+ SeenDbgUsers.clear();
+ SeenDbgVars.clear();
+
return MadeChange;
}
+void MachineSinking::ProcessDbgInst(MachineInstr &MI) {
+ // When we see DBG_VALUEs for registers, record any vreg it reads, so that
+ // we know what to sink if the vreg def sinks.
+ assert(MI.isDebugValue() && "Expected DBG_VALUE for processing");
+
+ DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
+ MI.getDebugLoc()->getInlinedAt());
+ bool SeenBefore = SeenDbgVars.count(Var) != 0;
+
+ MachineOperand &MO = MI.getOperand(0);
+ if (MO.isReg() && MO.getReg().isVirtual())
+ SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore));
+
+ // Record the variable for any DBG_VALUE, to avoid re-ordering any of them.
+ SeenDbgVars.insert(Var);
+}
+
bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr &MI,
MachineBasicBlock *From,
MachineBasicBlock *To) {
@@ -416,13 +464,13 @@ bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr &MI,
const MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg() || !MO.isUse())
continue;
- unsigned Reg = MO.getReg();
+ Register Reg = MO.getReg();
if (Reg == 0)
continue;
// We don't move live definitions of physical registers,
// so sinking their uses won't enable any opportunities.
- if (TargetRegisterInfo::isPhysicalRegister(Reg))
+ if (Register::isPhysicalRegister(Reg))
continue;
// If this instruction is the only user of a virtual register,
@@ -615,10 +663,10 @@ MachineSinking::FindSuccToSinkTo(MachineInstr &MI, MachineBasicBlock *MBB,
const MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg()) continue; // Ignore non-register operands.
- unsigned Reg = MO.getReg();
+ Register Reg = MO.getReg();
if (Reg == 0) continue;
- if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
+ if (Register::isPhysicalRegister(Reg)) {
if (MO.isUse()) {
// If the physreg has no defs anywhere, it's just an ambient register
// and we can freely move its uses. Alternatively, if it's allocatable,
@@ -735,18 +783,60 @@ static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI,
MBP.LHS.getReg() == BaseOp->getReg();
}
-/// Sink an instruction and its associated debug instructions. If the debug
-/// instructions to be sunk are already known, they can be provided in DbgVals.
+/// If the sunk instruction is a copy, try to forward the copy instead of
+/// leaving an 'undef' DBG_VALUE in the original location. Don't do this if
+/// there's any subregister weirdness involved. Returns true if copy
+/// propagation occurred.
+static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) {
+ const MachineRegisterInfo &MRI = SinkInst.getMF()->getRegInfo();
+ const TargetInstrInfo &TII = *SinkInst.getMF()->getSubtarget().getInstrInfo();
+
+ // Copy DBG_VALUE operand and set the original to undef. We then check to
+ // see whether this is something that can be copy-forwarded. If it isn't,
+ // continue around the loop.
+ MachineOperand DbgMO = DbgMI.getOperand(0);
+
+ const MachineOperand *SrcMO = nullptr, *DstMO = nullptr;
+ auto CopyOperands = TII.isCopyInstr(SinkInst);
+ if (!CopyOperands)
+ return false;
+ SrcMO = CopyOperands->Source;
+ DstMO = CopyOperands->Destination;
+
+ // Check validity of forwarding this copy.
+ bool PostRA = MRI.getNumVirtRegs() == 0;
+
+ // Trying to forward between physical and virtual registers is too hard.
+ if (DbgMO.getReg().isVirtual() != SrcMO->getReg().isVirtual())
+ return false;
+
+ // Only try virtual register copy-forwarding before regalloc, and physical
+ // register copy-forwarding after regalloc.
+ bool arePhysRegs = !DbgMO.getReg().isVirtual();
+ if (arePhysRegs != PostRA)
+ return false;
+
+ // Pre-regalloc, only forward if all subregisters agree (or there are no
+ // subregs at all). More analysis might recover some forwardable copies.
+ if (!PostRA && (DbgMO.getSubReg() != SrcMO->getSubReg() ||
+ DbgMO.getSubReg() != DstMO->getSubReg()))
+ return false;
+
+ // Post-regalloc, we may be sinking a DBG_VALUE of a sub or super-register
+ // of this copy. Only forward the copy if the DBG_VALUE operand exactly
+ // matches the copy destination.
+ if (PostRA && DbgMO.getReg() != DstMO->getReg())
+ return false;
+
+ DbgMI.getOperand(0).setReg(SrcMO->getReg());
+ DbgMI.getOperand(0).setSubReg(SrcMO->getSubReg());
+ return true;
+}
+
+/// Sink an instruction and its associated debug instructions.
static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
MachineBasicBlock::iterator InsertPos,
- SmallVectorImpl<MachineInstr *> *DbgVals = nullptr) {
- // If debug values are provided use those, otherwise call collectDebugValues.
- SmallVector<MachineInstr *, 2> DbgValuesToSink;
- if (DbgVals)
- DbgValuesToSink.insert(DbgValuesToSink.begin(),
- DbgVals->begin(), DbgVals->end());
- else
- MI.collectDebugValues(DbgValuesToSink);
+ SmallVectorImpl<MachineInstr *> &DbgValuesToSink) {
// If we cannot find a location to use (merge with), then we erase the debug
// location to prevent debug-info driven tools from potentially reporting
@@ -762,13 +852,19 @@ static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
SuccToSinkTo.splice(InsertPos, ParentBlock, MI,
++MachineBasicBlock::iterator(MI));
- // Move previously adjacent debug value instructions to the insert position.
+ // Sink a copy of debug users to the insert position. Mark the original
+ // DBG_VALUE location as 'undef', indicating that any earlier variable
+ // location should be terminated as we've optimised away the value at this
+ // point.
for (SmallVectorImpl<MachineInstr *>::iterator DBI = DbgValuesToSink.begin(),
DBE = DbgValuesToSink.end();
DBI != DBE; ++DBI) {
MachineInstr *DbgMI = *DBI;
- SuccToSinkTo.splice(InsertPos, ParentBlock, DbgMI,
- ++MachineBasicBlock::iterator(DbgMI));
+ MachineInstr *NewDbgMI = DbgMI->getMF()->CloneMachineInstr(*DBI);
+ SuccToSinkTo.insert(InsertPos, NewDbgMI);
+
+ if (!attemptDebugCopyProp(MI, *DbgMI))
+ DbgMI->getOperand(0).setReg(0);
}
}
@@ -817,8 +913,9 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
const MachineOperand &MO = MI.getOperand(I);
if (!MO.isReg()) continue;
- unsigned Reg = MO.getReg();
- if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
+ Register Reg = MO.getReg();
+ if (Reg == 0 || !Register::isPhysicalRegister(Reg))
+ continue;
if (SuccToSinkTo->isLiveIn(Reg))
return false;
}
@@ -885,7 +982,36 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
++InsertPos;
- performSink(MI, *SuccToSinkTo, InsertPos);
+ // Collect debug users of any vreg that this inst defines.
+ SmallVector<MachineInstr *, 4> DbgUsersToSink;
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
+ continue;
+ if (!SeenDbgUsers.count(MO.getReg()))
+ continue;
+
+ // Sink any users that don't pass any other DBG_VALUEs for this variable.
+ auto &Users = SeenDbgUsers[MO.getReg()];
+ for (auto &User : Users) {
+ MachineInstr *DbgMI = User.getPointer();
+ if (User.getInt()) {
+ // This DBG_VALUE would re-order assignments. If we can't copy-propagate
+ // it, it can't be recovered. Set it undef.
+ if (!attemptDebugCopyProp(MI, *DbgMI))
+ DbgMI->getOperand(0).setReg(0);
+ } else {
+ DbgUsersToSink.push_back(DbgMI);
+ }
+ }
+ }
+
+ // After sinking, some debug users may not be dominated any more. If possible,
+ // copy-propagate their operands. As it's expensive, don't do this if there's
+ // no debuginfo in the program.
+ if (MI.getMF()->getFunction().getSubprogram() && MI.isCopy())
+ SalvageUnsunkDebugUsersOfCopy(MI, SuccToSinkTo);
+
+ performSink(MI, *SuccToSinkTo, InsertPos, DbgUsersToSink);
// Conservatively, clear any kill flags, since it's possible that they are no
// longer correct.
@@ -900,6 +1026,41 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
return true;
}
+void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
+ MachineInstr &MI, MachineBasicBlock *TargetBlock) {
+ assert(MI.isCopy());
+ assert(MI.getOperand(1).isReg());
+
+ // Enumerate all users of vreg operands that are def'd. Skip those that will
+ // be sunk. For the rest, if they are not dominated by the block we will sink
+ // MI into, propagate the copy source to them.
+ SmallVector<MachineInstr *, 4> DbgDefUsers;
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
+ continue;
+ for (auto &User : MRI.use_instructions(MO.getReg())) {
+ if (!User.isDebugValue() || DT->dominates(TargetBlock, User.getParent()))
+ continue;
+
+ // If is in same block, will either sink or be use-before-def.
+ if (User.getParent() == MI.getParent())
+ continue;
+
+ assert(User.getOperand(0).isReg() &&
+ "DBG_VALUE user of vreg, but non reg operand?");
+ DbgDefUsers.push_back(&User);
+ }
+ }
+
+ // Point the users of this copy that are no longer dominated, at the source
+ // of the copy.
+ for (auto *User : DbgDefUsers) {
+ User->getOperand(0).setReg(MI.getOperand(1).getReg());
+ User->getOperand(0).setSubReg(MI.getOperand(1).getSubReg());
+ }
+}
+
//===----------------------------------------------------------------------===//
// This pass is not intended to be a replacement or a complete alternative
// for the pre-ra machine sink pass. It is only designed to sink COPY
@@ -958,8 +1119,9 @@ private:
/// Track which register units have been modified and used.
LiveRegUnits ModifiedRegUnits, UsedRegUnits;
- /// Track DBG_VALUEs of (unmodified) register units.
- DenseMap<unsigned, TinyPtrVector<MachineInstr*>> SeenDbgInstrs;
+ /// Track DBG_VALUEs of (unmodified) register units. Each DBG_VALUE has an
+ /// entry in this map for each unit it touches.
+ DenseMap<unsigned, TinyPtrVector<MachineInstr *>> SeenDbgInstrs;
/// Sink Copy instructions unused in the same block close to their uses in
/// successors.
@@ -1030,7 +1192,7 @@ static void clearKillFlags(MachineInstr *MI, MachineBasicBlock &CurBB,
const TargetRegisterInfo *TRI) {
for (auto U : UsedOpsInCopy) {
MachineOperand &MO = MI->getOperand(U);
- unsigned SrcReg = MO.getReg();
+ Register SrcReg = MO.getReg();
if (!UsedRegUnits.available(SrcReg)) {
MachineBasicBlock::iterator NI = std::next(MI->getIterator());
for (MachineInstr &UI : make_range(NI, CurBB.end())) {
@@ -1053,10 +1215,14 @@ static void updateLiveIn(MachineInstr *MI, MachineBasicBlock *SuccBB,
for (MCSubRegIterator S(DefReg, TRI, true); S.isValid(); ++S)
SuccBB->removeLiveIn(*S);
for (auto U : UsedOpsInCopy) {
- unsigned Reg = MI->getOperand(U).getReg();
- if (!SuccBB->isLiveIn(Reg))
- SuccBB->addLiveIn(Reg);
+ Register SrcReg = MI->getOperand(U).getReg();
+ LaneBitmask Mask;
+ for (MCRegUnitMaskIterator S(SrcReg, TRI); S.isValid(); ++S) {
+ Mask |= (*S).second;
+ }
+ SuccBB->addLiveIn(SrcReg, Mask.any() ? Mask : LaneBitmask::getAll());
}
+ SuccBB->sortUniqueLiveIns();
}
static bool hasRegisterDependency(MachineInstr *MI,
@@ -1069,7 +1235,7 @@ static bool hasRegisterDependency(MachineInstr *MI,
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg())
continue;
- unsigned Reg = MO.getReg();
+ Register Reg = MO.getReg();
if (!Reg)
continue;
if (MO.isDef()) {
@@ -1094,6 +1260,14 @@ static bool hasRegisterDependency(MachineInstr *MI,
return HasRegDependency;
}
+static SmallSet<unsigned, 4> getRegUnits(unsigned Reg,
+ const TargetRegisterInfo *TRI) {
+ SmallSet<unsigned, 4> RegUnits;
+ for (auto RI = MCRegUnitIterator(Reg, TRI); RI.isValid(); ++RI)
+ RegUnits.insert(*RI);
+ return RegUnits;
+}
+
bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
MachineFunction &MF,
const TargetRegisterInfo *TRI,
@@ -1130,15 +1304,17 @@ bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
// for DBG_VALUEs later, record them when they're encountered.
if (MI->isDebugValue()) {
auto &MO = MI->getOperand(0);
- if (MO.isReg() && TRI->isPhysicalRegister(MO.getReg())) {
+ if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) {
// Bail if we can already tell the sink would be rejected, rather
// than needlessly accumulating lots of DBG_VALUEs.
if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy,
ModifiedRegUnits, UsedRegUnits))
continue;
- // Record debug use of this register.
- SeenDbgInstrs[MO.getReg()].push_back(MI);
+ // Record debug use of each reg unit.
+ SmallSet<unsigned, 4> Units = getRegUnits(MO.getReg(), TRI);
+ for (unsigned Reg : Units)
+ SeenDbgInstrs[Reg].push_back(MI);
}
continue;
}
@@ -1177,21 +1353,28 @@ bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
assert((SuccBB->pred_size() == 1 && *SuccBB->pred_begin() == &CurBB) &&
"Unexpected predecessor");
- // Collect DBG_VALUEs that must sink with this copy.
+ // Collect DBG_VALUEs that must sink with this copy. We've previously
+ // recorded which reg units that DBG_VALUEs read, if this instruction
+ // writes any of those units then the corresponding DBG_VALUEs must sink.
+ SetVector<MachineInstr *> DbgValsToSinkSet;
SmallVector<MachineInstr *, 4> DbgValsToSink;
for (auto &MO : MI->operands()) {
if (!MO.isReg() || !MO.isDef())
continue;
- unsigned reg = MO.getReg();
- for (auto *MI : SeenDbgInstrs.lookup(reg))
- DbgValsToSink.push_back(MI);
+
+ SmallSet<unsigned, 4> Units = getRegUnits(MO.getReg(), TRI);
+ for (unsigned Reg : Units)
+ for (auto *MI : SeenDbgInstrs.lookup(Reg))
+ DbgValsToSinkSet.insert(MI);
}
+ DbgValsToSink.insert(DbgValsToSink.begin(), DbgValsToSinkSet.begin(),
+ DbgValsToSinkSet.end());
// Clear the kill flag if SrcReg is killed between MI and the end of the
// block.
clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI);
MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI();
- performSink(*MI, *SuccBB, InsertPos, &DbgValsToSink);
+ performSink(*MI, *SuccBB, InsertPos, DbgValsToSink);
updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy);
Changed = true;
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-04 11:47:32 +0000