summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp127
1 files changed, 65 insertions, 62 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
index ae1170ad1be6..e42701b9c6ca 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -48,6 +48,7 @@
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/Config/llvm-config.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/LaneBitmask.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
@@ -82,6 +83,10 @@ cl::opt<bool>
DumpCriticalPathLength("misched-dcpl", cl::Hidden,
cl::desc("Print critical path length to stdout"));
+cl::opt<bool> VerifyScheduling(
+ "verify-misched", cl::Hidden,
+ cl::desc("Verify machine instrs before and after machine scheduling"));
+
} // end namespace llvm
#ifndef NDEBUG
@@ -122,9 +127,6 @@ static cl::opt<bool> EnableMemOpCluster("misched-cluster", cl::Hidden,
cl::desc("Enable memop clustering."),
cl::init(true));
-static cl::opt<bool> VerifyScheduling("verify-misched", cl::Hidden,
- cl::desc("Verify machine instrs before and after machine scheduling"));
-
// DAG subtrees must have at least this many nodes.
static const unsigned MinSubtreeSize = 8;
@@ -198,6 +200,7 @@ char &llvm::MachineSchedulerID = MachineScheduler::ID;
INITIALIZE_PASS_BEGIN(MachineScheduler, DEBUG_TYPE,
"Machine Instruction Scheduler", false, false)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
@@ -210,7 +213,7 @@ MachineScheduler::MachineScheduler() : MachineSchedulerBase(ID) {
void MachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
- AU.addRequiredID(MachineDominatorsID);
+ AU.addRequired<MachineDominatorTree>();
AU.addRequired<MachineLoopInfo>();
AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<TargetPassConfig>();
@@ -234,8 +237,9 @@ PostMachineScheduler::PostMachineScheduler() : MachineSchedulerBase(ID) {
void PostMachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
- AU.addRequiredID(MachineDominatorsID);
+ AU.addRequired<MachineDominatorTree>();
AU.addRequired<MachineLoopInfo>();
+ AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<TargetPassConfig>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -400,7 +404,7 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
if (EnablePostRAMachineSched.getNumOccurrences()) {
if (!EnablePostRAMachineSched)
return false;
- } else if (!mf.getSubtarget().enablePostRAScheduler()) {
+ } else if (!mf.getSubtarget().enablePostRAMachineScheduler()) {
LLVM_DEBUG(dbgs() << "Subtarget disables post-MI-sched.\n");
return false;
}
@@ -410,6 +414,7 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MLI = &getAnalysis<MachineLoopInfo>();
PassConfig = &getAnalysis<TargetPassConfig>();
+ AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
if (VerifyScheduling)
MF->verify(this, "Before post machine scheduling.");
@@ -933,8 +938,8 @@ void ScheduleDAGMILive::collectVRegUses(SUnit &SU) {
if (TrackLaneMasks && !MO.isUse())
continue;
- unsigned Reg = MO.getReg();
- if (!TargetRegisterInfo::isVirtualRegister(Reg))
+ Register Reg = MO.getReg();
+ if (!Register::isVirtualRegister(Reg))
continue;
// Ignore re-defs.
@@ -985,7 +990,7 @@ void ScheduleDAGMILive::enterRegion(MachineBasicBlock *bb,
"ShouldTrackLaneMasks requires ShouldTrackPressure");
}
-// Setup the register pressure trackers for the top scheduled top and bottom
+// Setup the register pressure trackers for the top scheduled and bottom
// scheduled regions.
void ScheduleDAGMILive::initRegPressure() {
VRegUses.clear();
@@ -1095,7 +1100,7 @@ void ScheduleDAGMILive::updatePressureDiffs(
for (const RegisterMaskPair &P : LiveUses) {
unsigned Reg = P.RegUnit;
/// FIXME: Currently assuming single-use physregs.
- if (!TRI->isVirtualRegister(Reg))
+ if (!Register::isVirtualRegister(Reg))
continue;
if (ShouldTrackLaneMasks) {
@@ -1319,8 +1324,8 @@ unsigned ScheduleDAGMILive::computeCyclicCriticalPath() {
// Visit each live out vreg def to find def/use pairs that cross iterations.
for (const RegisterMaskPair &P : RPTracker.getPressure().LiveOutRegs) {
unsigned Reg = P.RegUnit;
- if (!TRI->isVirtualRegister(Reg))
- continue;
+ if (!Register::isVirtualRegister(Reg))
+ continue;
const LiveInterval &LI = LIS->getInterval(Reg);
const VNInfo *DefVNI = LI.getVNInfoBefore(LIS->getMBBEndIdx(BB));
if (!DefVNI)
@@ -1493,7 +1498,7 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
: BaseOp->getIndex() < RHS.BaseOp->getIndex();
if (Offset != RHS.Offset)
- return StackGrowsDown ? Offset > RHS.Offset : Offset < RHS.Offset;
+ return Offset < RHS.Offset;
return SU->NodeNum < RHS.SU->NodeNum;
}
@@ -1538,14 +1543,14 @@ namespace llvm {
std::unique_ptr<ScheduleDAGMutation>
createLoadClusterDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return EnableMemOpCluster ? llvm::make_unique<LoadClusterMutation>(TII, TRI)
+ return EnableMemOpCluster ? std::make_unique<LoadClusterMutation>(TII, TRI)
: nullptr;
}
std::unique_ptr<ScheduleDAGMutation>
createStoreClusterDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return EnableMemOpCluster ? llvm::make_unique<StoreClusterMutation>(TII, TRI)
+ return EnableMemOpCluster ? std::make_unique<StoreClusterMutation>(TII, TRI)
: nullptr;
}
@@ -1568,6 +1573,8 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
for (unsigned Idx = 0, End = MemOpRecords.size(); Idx < (End - 1); ++Idx) {
SUnit *SUa = MemOpRecords[Idx].SU;
SUnit *SUb = MemOpRecords[Idx+1].SU;
+ if (SUa->NodeNum > SUb->NodeNum)
+ std::swap(SUa, SUb);
if (TII->shouldClusterMemOps(*MemOpRecords[Idx].BaseOp,
*MemOpRecords[Idx + 1].BaseOp,
ClusterLength) &&
@@ -1593,10 +1600,8 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
/// Callback from DAG postProcessing to create cluster edges for loads.
void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
- // Map DAG NodeNum to store chain ID.
- DenseMap<unsigned, unsigned> StoreChainIDs;
- // Map each store chain to a set of dependent MemOps.
- SmallVector<SmallVector<SUnit*,4>, 32> StoreChainDependents;
+ // Map DAG NodeNum to a set of dependent MemOps in store chain.
+ DenseMap<unsigned, SmallVector<SUnit *, 4>> StoreChains;
for (SUnit &SU : DAG->SUnits) {
if ((IsLoad && !SU.getInstr()->mayLoad()) ||
(!IsLoad && !SU.getInstr()->mayStore()))
@@ -1609,19 +1614,14 @@ void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
break;
}
}
- // Check if this chain-like pred has been seen
- // before. ChainPredID==MaxNodeID at the top of the schedule.
- unsigned NumChains = StoreChainDependents.size();
- std::pair<DenseMap<unsigned, unsigned>::iterator, bool> Result =
- StoreChainIDs.insert(std::make_pair(ChainPredID, NumChains));
- if (Result.second)
- StoreChainDependents.resize(NumChains + 1);
- StoreChainDependents[Result.first->second].push_back(&SU);
+ // Insert the SU to corresponding store chain.
+ auto &Chain = StoreChains.FindAndConstruct(ChainPredID).second;
+ Chain.push_back(&SU);
}
// Iterate over the store chains.
- for (auto &SCD : StoreChainDependents)
- clusterNeighboringMemOps(SCD, DAG);
+ for (auto &SCD : StoreChains)
+ clusterNeighboringMemOps(SCD.second, DAG);
}
//===----------------------------------------------------------------------===//
@@ -1657,7 +1657,7 @@ namespace llvm {
std::unique_ptr<ScheduleDAGMutation>
createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return llvm::make_unique<CopyConstrain>(TII, TRI);
+ return std::make_unique<CopyConstrain>(TII, TRI);
}
} // end namespace llvm
@@ -1687,13 +1687,13 @@ void CopyConstrain::constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG) {
// Check for pure vreg copies.
const MachineOperand &SrcOp = Copy->getOperand(1);
- unsigned SrcReg = SrcOp.getReg();
- if (!TargetRegisterInfo::isVirtualRegister(SrcReg) || !SrcOp.readsReg())
+ Register SrcReg = SrcOp.getReg();
+ if (!Register::isVirtualRegister(SrcReg) || !SrcOp.readsReg())
return;
const MachineOperand &DstOp = Copy->getOperand(0);
- unsigned DstReg = DstOp.getReg();
- if (!TargetRegisterInfo::isVirtualRegister(DstReg) || DstOp.isDead())
+ Register DstReg = DstOp.getReg();
+ if (!Register::isVirtualRegister(DstReg) || DstOp.isDead())
return;
// Check if either the dest or source is local. If it's live across a back
@@ -2083,7 +2083,8 @@ getOtherResourceCount(unsigned &OtherCritIdx) {
return OtherCritCount;
}
-void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) {
+void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue,
+ unsigned Idx) {
assert(SU->getInstr() && "Scheduled SUnit must have instr");
#ifndef NDEBUG
@@ -2100,11 +2101,19 @@ void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) {
// Check for interlocks first. For the purpose of other heuristics, an
// instruction that cannot issue appears as if it's not in the ReadyQueue.
bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0;
- if ((!IsBuffered && ReadyCycle > CurrCycle) || checkHazard(SU) ||
- Available.size() >= ReadyListLimit)
- Pending.push(SU);
- else
+ bool HazardDetected = (!IsBuffered && ReadyCycle > CurrCycle) ||
+ checkHazard(SU) || (Available.size() >= ReadyListLimit);
+
+ if (!HazardDetected) {
Available.push(SU);
+
+ if (InPQueue)
+ Pending.remove(Pending.begin() + Idx);
+ return;
+ }
+
+ if (!InPQueue)
+ Pending.push(SU);
}
/// Move the boundary of scheduled code by one cycle.
@@ -2344,26 +2353,21 @@ void SchedBoundary::releasePending() {
// Check to see if any of the pending instructions are ready to issue. If
// so, add them to the available queue.
- bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0;
- for (unsigned i = 0, e = Pending.size(); i != e; ++i) {
- SUnit *SU = *(Pending.begin()+i);
+ for (unsigned I = 0, E = Pending.size(); I < E; ++I) {
+ SUnit *SU = *(Pending.begin() + I);
unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
if (ReadyCycle < MinReadyCycle)
MinReadyCycle = ReadyCycle;
- if (!IsBuffered && ReadyCycle > CurrCycle)
- continue;
-
- if (checkHazard(SU))
- continue;
-
if (Available.size() >= ReadyListLimit)
break;
- Available.push(SU);
- Pending.remove(Pending.begin()+i);
- --i; --e;
+ releaseNode(SU, ReadyCycle, true, I);
+ if (E != Pending.size()) {
+ --I;
+ --E;
+ }
}
CheckPending = false;
}
@@ -2914,14 +2918,12 @@ int biasPhysReg(const SUnit *SU, bool isTop) {
unsigned UnscheduledOper = isTop ? 0 : 1;
// If we have already scheduled the physreg produce/consumer, immediately
// schedule the copy.
- if (TargetRegisterInfo::isPhysicalRegister(
- MI->getOperand(ScheduledOper).getReg()))
+ if (Register::isPhysicalRegister(MI->getOperand(ScheduledOper).getReg()))
return 1;
// If the physreg is at the boundary, defer it. Otherwise schedule it
// immediately to free the dependent. We can hoist the copy later.
bool AtBoundary = isTop ? !SU->NumSuccsLeft : !SU->NumPredsLeft;
- if (TargetRegisterInfo::isPhysicalRegister(
- MI->getOperand(UnscheduledOper).getReg()))
+ if (Register::isPhysicalRegister(MI->getOperand(UnscheduledOper).getReg()))
return AtBoundary ? -1 : 1;
}
@@ -2931,7 +2933,7 @@ int biasPhysReg(const SUnit *SU, bool isTop) {
// physical registers.
bool DoBias = true;
for (const MachineOperand &Op : MI->defs()) {
- if (Op.isReg() && !TargetRegisterInfo::isPhysicalRegister(Op.getReg())) {
+ if (Op.isReg() && !Register::isPhysicalRegister(Op.getReg())) {
DoBias = false;
break;
}
@@ -3259,7 +3261,8 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
// Find already scheduled copies with a single physreg dependence and move
// them just above the scheduled instruction.
for (SDep &Dep : Deps) {
- if (Dep.getKind() != SDep::Data || !TRI->isPhysicalRegister(Dep.getReg()))
+ if (Dep.getKind() != SDep::Data ||
+ !Register::isPhysicalRegister(Dep.getReg()))
continue;
SUnit *DepSU = Dep.getSUnit();
if (isTop ? DepSU->Succs.size() > 1 : DepSU->Preds.size() > 1)
@@ -3298,7 +3301,7 @@ void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
/// default scheduler if the target does not set a default.
ScheduleDAGMILive *llvm::createGenericSchedLive(MachineSchedContext *C) {
ScheduleDAGMILive *DAG =
- new ScheduleDAGMILive(C, llvm::make_unique<GenericScheduler>(C));
+ new ScheduleDAGMILive(C, std::make_unique<GenericScheduler>(C));
// Register DAG post-processors.
//
// FIXME: extend the mutation API to allow earlier mutations to instantiate
@@ -3450,7 +3453,7 @@ void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
}
ScheduleDAGMI *llvm::createGenericSchedPostRA(MachineSchedContext *C) {
- return new ScheduleDAGMI(C, llvm::make_unique<PostGenericScheduler>(C),
+ return new ScheduleDAGMI(C, std::make_unique<PostGenericScheduler>(C),
/*RemoveKillFlags=*/true);
}
@@ -3561,10 +3564,10 @@ public:
} // end anonymous namespace
static ScheduleDAGInstrs *createILPMaxScheduler(MachineSchedContext *C) {
- return new ScheduleDAGMILive(C, llvm::make_unique<ILPScheduler>(true));
+ return new ScheduleDAGMILive(C, std::make_unique<ILPScheduler>(true));
}
static ScheduleDAGInstrs *createILPMinScheduler(MachineSchedContext *C) {
- return new ScheduleDAGMILive(C, llvm::make_unique<ILPScheduler>(false));
+ return new ScheduleDAGMILive(C, std::make_unique<ILPScheduler>(false));
}
static MachineSchedRegistry ILPMaxRegistry(
@@ -3658,7 +3661,7 @@ static ScheduleDAGInstrs *createInstructionShuffler(MachineSchedContext *C) {
assert((TopDown || !ForceTopDown) &&
"-misched-topdown incompatible with -misched-bottomup");
return new ScheduleDAGMILive(
- C, llvm::make_unique<InstructionShuffler>(Alternate, TopDown));
+ C, std::make_unique<InstructionShuffler>(Alternate, TopDown));
}
static MachineSchedRegistry ShufflerRegistry(
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-06 20:25:35 +0000