summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp127
1 files changed, 62 insertions, 65 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
index e42701b9c6ca..ae1170ad1be6 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -48,7 +48,6 @@
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/Config/llvm-config.h"
-#include "llvm/InitializePasses.h"
#include "llvm/MC/LaneBitmask.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
@@ -83,10 +82,6 @@ cl::opt<bool>
DumpCriticalPathLength("misched-dcpl", cl::Hidden,
cl::desc("Print critical path length to stdout"));
-cl::opt<bool> VerifyScheduling(
- "verify-misched", cl::Hidden,
- cl::desc("Verify machine instrs before and after machine scheduling"));
-
} // end namespace llvm
#ifndef NDEBUG
@@ -127,6 +122,9 @@ static cl::opt<bool> EnableMemOpCluster("misched-cluster", cl::Hidden,
cl::desc("Enable memop clustering."),
cl::init(true));
+static cl::opt<bool> VerifyScheduling("verify-misched", cl::Hidden,
+ cl::desc("Verify machine instrs before and after machine scheduling"));
+
// DAG subtrees must have at least this many nodes.
static const unsigned MinSubtreeSize = 8;
@@ -200,7 +198,6 @@ char &llvm::MachineSchedulerID = MachineScheduler::ID;
INITIALIZE_PASS_BEGIN(MachineScheduler, DEBUG_TYPE,
"Machine Instruction Scheduler", false, false)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
@@ -213,7 +210,7 @@ MachineScheduler::MachineScheduler() : MachineSchedulerBase(ID) {
void MachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
- AU.addRequired<MachineDominatorTree>();
+ AU.addRequiredID(MachineDominatorsID);
AU.addRequired<MachineLoopInfo>();
AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<TargetPassConfig>();
@@ -237,9 +234,8 @@ PostMachineScheduler::PostMachineScheduler() : MachineSchedulerBase(ID) {
void PostMachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
- AU.addRequired<MachineDominatorTree>();
+ AU.addRequiredID(MachineDominatorsID);
AU.addRequired<MachineLoopInfo>();
- AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<TargetPassConfig>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -404,7 +400,7 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
if (EnablePostRAMachineSched.getNumOccurrences()) {
if (!EnablePostRAMachineSched)
return false;
- } else if (!mf.getSubtarget().enablePostRAMachineScheduler()) {
+ } else if (!mf.getSubtarget().enablePostRAScheduler()) {
LLVM_DEBUG(dbgs() << "Subtarget disables post-MI-sched.\n");
return false;
}
@@ -414,7 +410,6 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MLI = &getAnalysis<MachineLoopInfo>();
PassConfig = &getAnalysis<TargetPassConfig>();
- AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
if (VerifyScheduling)
MF->verify(this, "Before post machine scheduling.");
@@ -938,8 +933,8 @@ void ScheduleDAGMILive::collectVRegUses(SUnit &SU) {
if (TrackLaneMasks && !MO.isUse())
continue;
- Register Reg = MO.getReg();
- if (!Register::isVirtualRegister(Reg))
+ unsigned Reg = MO.getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(Reg))
continue;
// Ignore re-defs.
@@ -990,7 +985,7 @@ void ScheduleDAGMILive::enterRegion(MachineBasicBlock *bb,
"ShouldTrackLaneMasks requires ShouldTrackPressure");
}
-// Setup the register pressure trackers for the top scheduled and bottom
+// Setup the register pressure trackers for the top scheduled top and bottom
// scheduled regions.
void ScheduleDAGMILive::initRegPressure() {
VRegUses.clear();
@@ -1100,7 +1095,7 @@ void ScheduleDAGMILive::updatePressureDiffs(
for (const RegisterMaskPair &P : LiveUses) {
unsigned Reg = P.RegUnit;
/// FIXME: Currently assuming single-use physregs.
- if (!Register::isVirtualRegister(Reg))
+ if (!TRI->isVirtualRegister(Reg))
continue;
if (ShouldTrackLaneMasks) {
@@ -1324,8 +1319,8 @@ unsigned ScheduleDAGMILive::computeCyclicCriticalPath() {
// Visit each live out vreg def to find def/use pairs that cross iterations.
for (const RegisterMaskPair &P : RPTracker.getPressure().LiveOutRegs) {
unsigned Reg = P.RegUnit;
- if (!Register::isVirtualRegister(Reg))
- continue;
+ if (!TRI->isVirtualRegister(Reg))
+ continue;
const LiveInterval &LI = LIS->getInterval(Reg);
const VNInfo *DefVNI = LI.getVNInfoBefore(LIS->getMBBEndIdx(BB));
if (!DefVNI)
@@ -1498,7 +1493,7 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
: BaseOp->getIndex() < RHS.BaseOp->getIndex();
if (Offset != RHS.Offset)
- return Offset < RHS.Offset;
+ return StackGrowsDown ? Offset > RHS.Offset : Offset < RHS.Offset;
return SU->NodeNum < RHS.SU->NodeNum;
}
@@ -1543,14 +1538,14 @@ namespace llvm {
std::unique_ptr<ScheduleDAGMutation>
createLoadClusterDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return EnableMemOpCluster ? std::make_unique<LoadClusterMutation>(TII, TRI)
+ return EnableMemOpCluster ? llvm::make_unique<LoadClusterMutation>(TII, TRI)
: nullptr;
}
std::unique_ptr<ScheduleDAGMutation>
createStoreClusterDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return EnableMemOpCluster ? std::make_unique<StoreClusterMutation>(TII, TRI)
+ return EnableMemOpCluster ? llvm::make_unique<StoreClusterMutation>(TII, TRI)
: nullptr;
}
@@ -1573,8 +1568,6 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
for (unsigned Idx = 0, End = MemOpRecords.size(); Idx < (End - 1); ++Idx) {
SUnit *SUa = MemOpRecords[Idx].SU;
SUnit *SUb = MemOpRecords[Idx+1].SU;
- if (SUa->NodeNum > SUb->NodeNum)
- std::swap(SUa, SUb);
if (TII->shouldClusterMemOps(*MemOpRecords[Idx].BaseOp,
*MemOpRecords[Idx + 1].BaseOp,
ClusterLength) &&
@@ -1600,8 +1593,10 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
/// Callback from DAG postProcessing to create cluster edges for loads.
void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
- // Map DAG NodeNum to a set of dependent MemOps in store chain.
- DenseMap<unsigned, SmallVector<SUnit *, 4>> StoreChains;
+ // Map DAG NodeNum to store chain ID.
+ DenseMap<unsigned, unsigned> StoreChainIDs;
+ // Map each store chain to a set of dependent MemOps.
+ SmallVector<SmallVector<SUnit*,4>, 32> StoreChainDependents;
for (SUnit &SU : DAG->SUnits) {
if ((IsLoad && !SU.getInstr()->mayLoad()) ||
(!IsLoad && !SU.getInstr()->mayStore()))
@@ -1614,14 +1609,19 @@ void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) {
break;
}
}
- // Insert the SU to corresponding store chain.
- auto &Chain = StoreChains.FindAndConstruct(ChainPredID).second;
- Chain.push_back(&SU);
+ // Check if this chain-like pred has been seen
+ // before. ChainPredID==MaxNodeID at the top of the schedule.
+ unsigned NumChains = StoreChainDependents.size();
+ std::pair<DenseMap<unsigned, unsigned>::iterator, bool> Result =
+ StoreChainIDs.insert(std::make_pair(ChainPredID, NumChains));
+ if (Result.second)
+ StoreChainDependents.resize(NumChains + 1);
+ StoreChainDependents[Result.first->second].push_back(&SU);
}
// Iterate over the store chains.
- for (auto &SCD : StoreChains)
- clusterNeighboringMemOps(SCD.second, DAG);
+ for (auto &SCD : StoreChainDependents)
+ clusterNeighboringMemOps(SCD, DAG);
}
//===----------------------------------------------------------------------===//
@@ -1657,7 +1657,7 @@ namespace llvm {
std::unique_ptr<ScheduleDAGMutation>
createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
const TargetRegisterInfo *TRI) {
- return std::make_unique<CopyConstrain>(TII, TRI);
+ return llvm::make_unique<CopyConstrain>(TII, TRI);
}
} // end namespace llvm
@@ -1687,13 +1687,13 @@ void CopyConstrain::constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG) {
// Check for pure vreg copies.
const MachineOperand &SrcOp = Copy->getOperand(1);
- Register SrcReg = SrcOp.getReg();
- if (!Register::isVirtualRegister(SrcReg) || !SrcOp.readsReg())
+ unsigned SrcReg = SrcOp.getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(SrcReg) || !SrcOp.readsReg())
return;
const MachineOperand &DstOp = Copy->getOperand(0);
- Register DstReg = DstOp.getReg();
- if (!Register::isVirtualRegister(DstReg) || DstOp.isDead())
+ unsigned DstReg = DstOp.getReg();
+ if (!TargetRegisterInfo::isVirtualRegister(DstReg) || DstOp.isDead())
return;
// Check if either the dest or source is local. If it's live across a back
@@ -2083,8 +2083,7 @@ getOtherResourceCount(unsigned &OtherCritIdx) {
return OtherCritCount;
}
-void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue,
- unsigned Idx) {
+void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) {
assert(SU->getInstr() && "Scheduled SUnit must have instr");
#ifndef NDEBUG
@@ -2101,19 +2100,11 @@ void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue,
// Check for interlocks first. For the purpose of other heuristics, an
// instruction that cannot issue appears as if it's not in the ReadyQueue.
bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0;
- bool HazardDetected = (!IsBuffered && ReadyCycle > CurrCycle) ||
- checkHazard(SU) || (Available.size() >= ReadyListLimit);
-
- if (!HazardDetected) {
- Available.push(SU);
-
- if (InPQueue)
- Pending.remove(Pending.begin() + Idx);
- return;
- }
-
- if (!InPQueue)
+ if ((!IsBuffered && ReadyCycle > CurrCycle) || checkHazard(SU) ||
+ Available.size() >= ReadyListLimit)
Pending.push(SU);
+ else
+ Available.push(SU);
}
/// Move the boundary of scheduled code by one cycle.
@@ -2353,21 +2344,26 @@ void SchedBoundary::releasePending() {
// Check to see if any of the pending instructions are ready to issue. If
// so, add them to the available queue.
- for (unsigned I = 0, E = Pending.size(); I < E; ++I) {
- SUnit *SU = *(Pending.begin() + I);
+ bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0;
+ for (unsigned i = 0, e = Pending.size(); i != e; ++i) {
+ SUnit *SU = *(Pending.begin()+i);
unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
if (ReadyCycle < MinReadyCycle)
MinReadyCycle = ReadyCycle;
+ if (!IsBuffered && ReadyCycle > CurrCycle)
+ continue;
+
+ if (checkHazard(SU))
+ continue;
+
if (Available.size() >= ReadyListLimit)
break;
- releaseNode(SU, ReadyCycle, true, I);
- if (E != Pending.size()) {
- --I;
- --E;
- }
+ Available.push(SU);
+ Pending.remove(Pending.begin()+i);
+ --i; --e;
}
CheckPending = false;
}
@@ -2918,12 +2914,14 @@ int biasPhysReg(const SUnit *SU, bool isTop) {
unsigned UnscheduledOper = isTop ? 0 : 1;
// If we have already scheduled the physreg produce/consumer, immediately
// schedule the copy.
- if (Register::isPhysicalRegister(MI->getOperand(ScheduledOper).getReg()))
+ if (TargetRegisterInfo::isPhysicalRegister(
+ MI->getOperand(ScheduledOper).getReg()))
return 1;
// If the physreg is at the boundary, defer it. Otherwise schedule it
// immediately to free the dependent. We can hoist the copy later.
bool AtBoundary = isTop ? !SU->NumSuccsLeft : !SU->NumPredsLeft;
- if (Register::isPhysicalRegister(MI->getOperand(UnscheduledOper).getReg()))
+ if (TargetRegisterInfo::isPhysicalRegister(
+ MI->getOperand(UnscheduledOper).getReg()))
return AtBoundary ? -1 : 1;
}
@@ -2933,7 +2931,7 @@ int biasPhysReg(const SUnit *SU, bool isTop) {
// physical registers.
bool DoBias = true;
for (const MachineOperand &Op : MI->defs()) {
- if (Op.isReg() && !Register::isPhysicalRegister(Op.getReg())) {
+ if (Op.isReg() && !TargetRegisterInfo::isPhysicalRegister(Op.getReg())) {
DoBias = false;
break;
}
@@ -3261,8 +3259,7 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
// Find already scheduled copies with a single physreg dependence and move
// them just above the scheduled instruction.
for (SDep &Dep : Deps) {
- if (Dep.getKind() != SDep::Data ||
- !Register::isPhysicalRegister(Dep.getReg()))
+ if (Dep.getKind() != SDep::Data || !TRI->isPhysicalRegister(Dep.getReg()))
continue;
SUnit *DepSU = Dep.getSUnit();
if (isTop ? DepSU->Succs.size() > 1 : DepSU->Preds.size() > 1)
@@ -3301,7 +3298,7 @@ void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
/// default scheduler if the target does not set a default.
ScheduleDAGMILive *llvm::createGenericSchedLive(MachineSchedContext *C) {
ScheduleDAGMILive *DAG =
- new ScheduleDAGMILive(C, std::make_unique<GenericScheduler>(C));
+ new ScheduleDAGMILive(C, llvm::make_unique<GenericScheduler>(C));
// Register DAG post-processors.
//
// FIXME: extend the mutation API to allow earlier mutations to instantiate
@@ -3453,7 +3450,7 @@ void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
}
ScheduleDAGMI *llvm::createGenericSchedPostRA(MachineSchedContext *C) {
- return new ScheduleDAGMI(C, std::make_unique<PostGenericScheduler>(C),
+ return new ScheduleDAGMI(C, llvm::make_unique<PostGenericScheduler>(C),
/*RemoveKillFlags=*/true);
}
@@ -3564,10 +3561,10 @@ public:
} // end anonymous namespace
static ScheduleDAGInstrs *createILPMaxScheduler(MachineSchedContext *C) {
- return new ScheduleDAGMILive(C, std::make_unique<ILPScheduler>(true));
+ return new ScheduleDAGMILive(C, llvm::make_unique<ILPScheduler>(true));
}
static ScheduleDAGInstrs *createILPMinScheduler(MachineSchedContext *C) {
- return new ScheduleDAGMILive(C, std::make_unique<ILPScheduler>(false));
+ return new ScheduleDAGMILive(C, llvm::make_unique<ILPScheduler>(false));
}
static MachineSchedRegistry ILPMaxRegistry(
@@ -3661,7 +3658,7 @@ static ScheduleDAGInstrs *createInstructionShuffler(MachineSchedContext *C) {
assert((TopDown || !ForceTopDown) &&
"-misched-topdown incompatible with -misched-bottomup");
return new ScheduleDAGMILive(
- C, std::make_unique<InstructionShuffler>(Alternate, TopDown));
+ C, llvm::make_unique<InstructionShuffler>(Alternate, TopDown));
}
static MachineSchedRegistry ShufflerRegistry(
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-10 20:45:21 +0000