aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/ModuloSchedule.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/ModuloSchedule.cpp')
-rw-r--r--llvm/lib/CodeGen/ModuloSchedule.cpp88
1 files changed, 56 insertions, 32 deletions
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index f91a9d2c3a32..3245d9649be1 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -11,6 +11,7 @@
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCContext.h"
@@ -157,7 +158,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
SmallVector<MachineBasicBlock *, 4> EpilogBBs;
// Generate the epilog instructions to complete the pipeline.
- generateEpilog(MaxStageCount, KernelBB, VRMap, EpilogBBs, PrologBBs);
+ generateEpilog(MaxStageCount, KernelBB, BB, VRMap, EpilogBBs, PrologBBs);
// We need this step because the register allocation doesn't handle some
// situations well, so we insert copies to help out.
@@ -239,11 +240,9 @@ void ModuloScheduleExpander::generateProlog(unsigned LastStage,
/// Generate the pipeline epilog code. The epilog code finishes the iterations
/// that were started in either the prolog or the kernel. We create a basic
/// block for each stage that needs to complete.
-void ModuloScheduleExpander::generateEpilog(unsigned LastStage,
- MachineBasicBlock *KernelBB,
- ValueMapTy *VRMap,
- MBBVectorTy &EpilogBBs,
- MBBVectorTy &PrologBBs) {
+void ModuloScheduleExpander::generateEpilog(
+ unsigned LastStage, MachineBasicBlock *KernelBB, MachineBasicBlock *OrigBB,
+ ValueMapTy *VRMap, MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs) {
// We need to change the branch from the kernel to the first epilog block, so
// this call to analyze branch uses the kernel rather than the original BB.
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
@@ -313,7 +312,12 @@ void ModuloScheduleExpander::generateEpilog(unsigned LastStage,
// Create a branch to the new epilog from the kernel.
// Remove the original branch and add a new branch to the epilog.
TII->removeBranch(*KernelBB);
- TII->insertBranch(*KernelBB, KernelBB, EpilogStart, Cond, DebugLoc());
+ assert((OrigBB == TBB || OrigBB == FBB) &&
+ "Unable to determine looping branch direction");
+ if (OrigBB != TBB)
+ TII->insertBranch(*KernelBB, EpilogStart, KernelBB, Cond, DebugLoc());
+ else
+ TII->insertBranch(*KernelBB, KernelBB, EpilogStart, Cond, DebugLoc());
// Add a branch to the loop exit.
if (EpilogBBs.size() > 0) {
MachineBasicBlock *LastEpilogBB = EpilogBBs.back();
@@ -813,8 +817,8 @@ static void removePhis(MachineBasicBlock *BB, MachineBasicBlock *Incoming) {
break;
for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2)
if (MI.getOperand(i + 1).getMBB() == Incoming) {
- MI.RemoveOperand(i + 1);
- MI.RemoveOperand(i);
+ MI.removeOperand(i + 1);
+ MI.removeOperand(i);
break;
}
}
@@ -846,7 +850,7 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
Optional<bool> StaticallyGreater =
LoopInfo->createTripCountGreaterCondition(j + 1, *Prolog, Cond);
unsigned numAdded = 0;
- if (!StaticallyGreater.hasValue()) {
+ if (!StaticallyGreater) {
Prolog->addSuccessor(Epilog);
numAdded = TII->insertBranch(*Prolog, Epilog, LastPro, Cond, DebugLoc());
} else if (*StaticallyGreater == false) {
@@ -999,7 +1003,7 @@ MachineInstr *ModuloScheduleExpander::cloneAndChangeInstr(
}
/// Update the machine instruction with new virtual registers. This
-/// function may change the defintions and/or uses.
+/// function may change the definitions and/or uses.
void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
bool LastDef,
unsigned CurStageNum,
@@ -1159,8 +1163,17 @@ void ModuloScheduleExpander::rewriteScheduledInstr(
if (!InProlog && !Phi->isPHI() && StagePhi < StageSched)
ReplaceReg = NewReg;
if (ReplaceReg) {
- MRI.constrainRegClass(ReplaceReg, MRI.getRegClass(OldReg));
- UseOp.setReg(ReplaceReg);
+ const TargetRegisterClass *NRC =
+ MRI.constrainRegClass(ReplaceReg, MRI.getRegClass(OldReg));
+ if (NRC)
+ UseOp.setReg(ReplaceReg);
+ else {
+ Register SplitReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
+ BuildMI(*BB, UseMI, UseMI->getDebugLoc(), TII->get(TargetOpcode::COPY),
+ SplitReg)
+ .addReg(ReplaceReg);
+ UseOp.setReg(SplitReg);
+ }
}
}
}
@@ -1205,8 +1218,12 @@ void EliminateDeadPhis(MachineBasicBlock *MBB, MachineRegisterInfo &MRI,
MI.eraseFromParent();
Changed = true;
} else if (!KeepSingleSrcPhi && MI.getNumExplicitOperands() == 3) {
- MRI.constrainRegClass(MI.getOperand(1).getReg(),
- MRI.getRegClass(MI.getOperand(0).getReg()));
+ const TargetRegisterClass *ConstrainRegClass =
+ MRI.constrainRegClass(MI.getOperand(1).getReg(),
+ MRI.getRegClass(MI.getOperand(0).getReg()));
+ assert(ConstrainRegClass &&
+ "Expected a valid constrained register class!");
+ (void)ConstrainRegClass;
MRI.replaceRegWith(MI.getOperand(0).getReg(),
MI.getOperand(1).getReg());
if (LIS)
@@ -1404,7 +1421,7 @@ Register KernelRewriter::remapUse(Register Reg, MachineInstr &MI) {
while (DefaultI != Defaults.rend())
LoopReg = phi(LoopReg, *DefaultI++, MRI.getRegClass(Reg));
- if (IllegalPhiDefault.hasValue()) {
+ if (IllegalPhiDefault) {
// The consumer optionally consumes LoopProducer in the same iteration
// (because the producer is scheduled at an earlier cycle than the consumer)
// or the initial value. To facilitate this we create an illegal block here
@@ -1414,7 +1431,7 @@ Register KernelRewriter::remapUse(Register Reg, MachineInstr &MI) {
Register R = MRI.createVirtualRegister(RC);
MachineInstr *IllegalPhi =
BuildMI(*BB, MI, DebugLoc(), TII->get(TargetOpcode::PHI), R)
- .addReg(IllegalPhiDefault.getValue())
+ .addReg(*IllegalPhiDefault)
.addMBB(PreheaderBB) // Block choice is arbitrary and has no effect.
.addReg(LoopReg)
.addMBB(BB); // Block choice is arbitrary and has no effect.
@@ -1430,7 +1447,7 @@ Register KernelRewriter::remapUse(Register Reg, MachineInstr &MI) {
Register KernelRewriter::phi(Register LoopReg, Optional<Register> InitReg,
const TargetRegisterClass *RC) {
// If the init register is not undef, try and find an existing phi.
- if (InitReg.hasValue()) {
+ if (InitReg) {
auto I = Phis.find({LoopReg, InitReg.getValue()});
if (I != Phis.end())
return I->second;
@@ -1446,7 +1463,7 @@ Register KernelRewriter::phi(Register LoopReg, Optional<Register> InitReg,
auto I = UndefPhis.find(LoopReg);
if (I != UndefPhis.end()) {
Register R = I->second;
- if (!InitReg.hasValue())
+ if (!InitReg)
// Found a phi taking undef as input, and this input is undef so return
// without any more changes.
return R;
@@ -1454,7 +1471,10 @@ Register KernelRewriter::phi(Register LoopReg, Optional<Register> InitReg,
MachineInstr *MI = MRI.getVRegDef(R);
MI->getOperand(1).setReg(InitReg.getValue());
Phis.insert({{LoopReg, InitReg.getValue()}, R});
- MRI.constrainRegClass(R, MRI.getRegClass(InitReg.getValue()));
+ const TargetRegisterClass *ConstrainRegClass =
+ MRI.constrainRegClass(R, MRI.getRegClass(InitReg.getValue()));
+ assert(ConstrainRegClass && "Expected a valid constrained register class!");
+ (void)ConstrainRegClass;
UndefPhis.erase(I);
return R;
}
@@ -1463,14 +1483,18 @@ Register KernelRewriter::phi(Register LoopReg, Optional<Register> InitReg,
if (!RC)
RC = MRI.getRegClass(LoopReg);
Register R = MRI.createVirtualRegister(RC);
- if (InitReg.hasValue())
- MRI.constrainRegClass(R, MRI.getRegClass(*InitReg));
+ if (InitReg) {
+ const TargetRegisterClass *ConstrainRegClass =
+ MRI.constrainRegClass(R, MRI.getRegClass(*InitReg));
+ assert(ConstrainRegClass && "Expected a valid constrained register class!");
+ (void)ConstrainRegClass;
+ }
BuildMI(*BB, BB->getFirstNonPHI(), DebugLoc(), TII->get(TargetOpcode::PHI), R)
- .addReg(InitReg.hasValue() ? *InitReg : undef(RC))
+ .addReg(InitReg ? *InitReg : undef(RC))
.addMBB(PreheaderBB)
.addReg(LoopReg)
.addMBB(BB);
- if (!InitReg.hasValue())
+ if (!InitReg)
UndefPhis[LoopReg] = R;
else
Phis[{LoopReg, *InitReg}] = R;
@@ -1793,10 +1817,10 @@ void PeelingModuloScheduleExpander::peelPrologAndEpilogs() {
// Iterate in reverse order over all instructions, remapping as we go.
for (MachineBasicBlock *B : reverse(Blocks)) {
- for (auto I = B->getFirstInstrTerminator()->getReverseIterator();
+ for (auto I = B->instr_rbegin();
I != std::next(B->getFirstNonPHI()->getReverseIterator());) {
- MachineInstr *MI = &*I++;
- rewriteUsesOf(MI);
+ MachineBasicBlock::reverse_instr_iterator MI = I++;
+ rewriteUsesOf(&*MI);
}
}
for (auto *MI : IllegalPhisToDelete) {
@@ -1919,7 +1943,7 @@ void PeelingModuloScheduleExpander::fixupBranches() {
TII->removeBranch(*Prolog);
Optional<bool> StaticallyGreater =
LoopInfo->createTripCountGreaterCondition(TC, *Prolog, Cond);
- if (!StaticallyGreater.hasValue()) {
+ if (!StaticallyGreater) {
LLVM_DEBUG(dbgs() << "Dynamic: TC > " << TC << "\n");
// Dynamically branch based on Cond.
TII->insertBranch(*Prolog, Epilog, Fallthrough, Cond, DebugLoc());
@@ -1929,8 +1953,8 @@ void PeelingModuloScheduleExpander::fixupBranches() {
// blocks. Leave it to unreachable-block-elim to clean up.
Prolog->removeSuccessor(Fallthrough);
for (MachineInstr &P : Fallthrough->phis()) {
- P.RemoveOperand(2);
- P.RemoveOperand(1);
+ P.removeOperand(2);
+ P.removeOperand(1);
}
TII->insertUnconditionalBranch(*Prolog, Epilog, DebugLoc());
KernelDisposed = true;
@@ -1939,8 +1963,8 @@ void PeelingModuloScheduleExpander::fixupBranches() {
// Prolog always falls through; remove incoming values in epilog.
Prolog->removeSuccessor(Epilog);
for (MachineInstr &P : Epilog->phis()) {
- P.RemoveOperand(4);
- P.RemoveOperand(3);
+ P.removeOperand(4);
+ P.removeOperand(3);
}
}
}
generated by cgit v1.3 (git 2.53.0) at 2026-04-07 03:14:19 +0000