aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp316
1 files changed, 244 insertions, 72 deletions
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 36d52ac3ee89..5839e59b4d7f 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -48,28 +48,11 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
-#include "AMDGPUSubtarget.h"
-#include "SIInstrInfo.h"
+#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/LiveIntervals.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/Passes.h"
-#include "llvm/CodeGen/SlotIndexes.h"
-#include "llvm/CodeGen/TargetRegisterInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/Pass.h"
-#include <cassert>
-#include <iterator>
using namespace llvm;
@@ -99,6 +82,7 @@ private:
unsigned MovTermOpc;
unsigned Andn2TermOpc;
unsigned XorTermrOpc;
+ unsigned OrTermrOpc;
unsigned OrSaveExecOpc;
unsigned Exec;
@@ -106,14 +90,19 @@ private:
void emitElse(MachineInstr &MI);
void emitIfBreak(MachineInstr &MI);
void emitLoop(MachineInstr &MI);
- void emitEndCf(MachineInstr &MI);
+
+ MachineBasicBlock *emitEndCf(MachineInstr &MI);
+
+ void lowerInitExec(MachineBasicBlock *MBB, MachineInstr &MI);
void findMaskOperands(MachineInstr &MI, unsigned OpNo,
SmallVectorImpl<MachineOperand> &Src) const;
void combineMasks(MachineInstr &MI);
- void process(MachineInstr &MI);
+ bool removeMBBifRedundant(MachineBasicBlock &MBB);
+
+ MachineBasicBlock *process(MachineInstr &MI);
// Skip to the next instruction, ignoring debug instructions, and trivial
// block boundaries (blocks that have one (typically fallthrough) successor,
@@ -122,6 +111,19 @@ private:
skipIgnoreExecInstsTrivialSucc(MachineBasicBlock &MBB,
MachineBasicBlock::iterator It) const;
+ /// Find the insertion point for a new conditional branch.
+ MachineBasicBlock::iterator
+ skipToUncondBrOrEnd(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I) const {
+ assert(I->isTerminator());
+
+ // FIXME: What if we had multiple pre-existing conditional branches?
+ MachineBasicBlock::iterator End = MBB.end();
+ while (I != End && !I->isUnconditionalBranch())
+ ++I;
+ return I;
+ }
+
// Remove redundant SI_END_CF instructions.
void optimizeEndCf();
@@ -141,9 +143,6 @@ public:
AU.addPreserved<SlotIndexes>();
AU.addPreserved<LiveIntervals>();
AU.addPreservedID(LiveVariablesID);
- AU.addPreservedID(MachineLoopInfoID);
- AU.addPreservedID(MachineDominatorsID);
- AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
@@ -167,8 +166,7 @@ char &llvm::SILowerControlFlowID = SILowerControlFlow::ID;
static bool hasKill(const MachineBasicBlock *Begin,
const MachineBasicBlock *End, const SIInstrInfo *TII) {
DenseSet<const MachineBasicBlock*> Visited;
- SmallVector<MachineBasicBlock *, 4> Worklist(Begin->succ_begin(),
- Begin->succ_end());
+ SmallVector<MachineBasicBlock *, 4> Worklist(Begin->successors());
while (!Worklist.empty()) {
MachineBasicBlock *MBB = Worklist.pop_back_val();
@@ -275,6 +273,10 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) {
BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec)
.addReg(Tmp, RegState::Kill);
+ // Skip ahead to the unconditional branch in case there are other terminators
+ // present.
+ I = skipToUncondBrOrEnd(MBB, I);
+
// Insert the S_CBRANCH_EXECZ instruction which will be optimized later
// during SIRemoveShortExecBranches.
MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
@@ -315,44 +317,37 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
Register DstReg = MI.getOperand(0).getReg();
- bool ExecModified = MI.getOperand(3).getImm() != 0;
MachineBasicBlock::iterator Start = MBB.begin();
- // We are running before TwoAddressInstructions, and si_else's operands are
- // tied. In order to correctly tie the registers, split this into a copy of
- // the src like it does.
- Register CopyReg = MRI->createVirtualRegister(BoolRC);
- MachineInstr *CopyExec =
- BuildMI(MBB, Start, DL, TII->get(AMDGPU::COPY), CopyReg)
- .add(MI.getOperand(1)); // Saved EXEC
-
// This must be inserted before phis and any spill code inserted before the
// else.
- Register SaveReg = ExecModified ?
- MRI->createVirtualRegister(BoolRC) : DstReg;
+ Register SaveReg = MRI->createVirtualRegister(BoolRC);
MachineInstr *OrSaveExec =
BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg)
- .addReg(CopyReg);
+ .add(MI.getOperand(1)); // Saved EXEC
MachineBasicBlock *DestBB = MI.getOperand(2).getMBB();
MachineBasicBlock::iterator ElsePt(MI);
- if (ExecModified) {
- MachineInstr *And =
- BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg)
- .addReg(Exec)
- .addReg(SaveReg);
+ // This accounts for any modification of the EXEC mask within the block and
+ // can be optimized out pre-RA when not required.
+ MachineInstr *And = BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg)
+ .addReg(Exec)
+ .addReg(SaveReg);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*And);
- }
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*And);
MachineInstr *Xor =
BuildMI(MBB, ElsePt, DL, TII->get(XorTermrOpc), Exec)
.addReg(Exec)
.addReg(DstReg);
+ // Skip ahead to the unconditional branch in case there are other terminators
+ // present.
+ ElsePt = skipToUncondBrOrEnd(MBB, ElsePt);
+
MachineInstr *Branch =
BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
.addMBB(DestBB);
@@ -365,18 +360,14 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
LIS->RemoveMachineInstrFromMaps(MI);
MI.eraseFromParent();
- LIS->InsertMachineInstrInMaps(*CopyExec);
LIS->InsertMachineInstrInMaps(*OrSaveExec);
LIS->InsertMachineInstrInMaps(*Xor);
LIS->InsertMachineInstrInMaps(*Branch);
- // src reg is tied to dst reg.
LIS->removeInterval(DstReg);
LIS->createAndComputeVirtRegInterval(DstReg);
- LIS->createAndComputeVirtRegInterval(CopyReg);
- if (ExecModified)
- LIS->createAndComputeVirtRegInterval(SaveReg);
+ LIS->createAndComputeVirtRegInterval(SaveReg);
// Let this be recomputed.
LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
@@ -435,8 +426,9 @@ void SILowerControlFlow::emitLoop(MachineInstr &MI) {
.addReg(Exec)
.add(MI.getOperand(0));
+ auto BranchPt = skipToUncondBrOrEnd(MBB, MI.getIterator());
MachineInstr *Branch =
- BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
+ BuildMI(MBB, BranchPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
.add(MI.getOperand(1));
if (LIS) {
@@ -479,19 +471,37 @@ SILowerControlFlow::skipIgnoreExecInstsTrivialSucc(
} while (true);
}
-void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
+MachineBasicBlock *SILowerControlFlow::emitEndCf(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
- MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
- unsigned CFMask = MI.getOperand(0).getReg();
- MachineInstr *Def = MRI.getUniqueVRegDef(CFMask);
const DebugLoc &DL = MI.getDebugLoc();
- MachineBasicBlock::iterator InsPt =
- Def && Def->getParent() == &MBB ? std::next(MachineBasicBlock::iterator(Def))
- : MBB.begin();
- MachineInstr *NewMI = BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec)
- .addReg(Exec)
- .add(MI.getOperand(0));
+ MachineBasicBlock::iterator InsPt = MBB.begin();
+
+ // If we have instructions that aren't prolog instructions, split the block
+ // and emit a terminator instruction. This ensures correct spill placement.
+ // FIXME: We should unconditionally split the block here.
+ bool NeedBlockSplit = false;
+ Register DataReg = MI.getOperand(0).getReg();
+ for (MachineBasicBlock::iterator I = InsPt, E = MI.getIterator();
+ I != E; ++I) {
+ if (I->modifiesRegister(DataReg, TRI)) {
+ NeedBlockSplit = true;
+ break;
+ }
+ }
+
+ unsigned Opcode = OrOpc;
+ MachineBasicBlock *SplitBB = &MBB;
+ if (NeedBlockSplit) {
+ SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/true, LIS);
+ Opcode = OrTermrOpc;
+ InsPt = MI;
+ }
+
+ MachineInstr *NewMI =
+ BuildMI(MBB, InsPt, DL, TII->get(Opcode), Exec)
+ .addReg(Exec)
+ .add(MI.getOperand(0));
LoweredEndCf.insert(NewMI);
@@ -512,6 +522,7 @@ void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
if (LIS)
LIS->handleMove(*NewMI);
+ return SplitBB;
}
// Returns replace operands for a logical operation, either single result
@@ -519,7 +530,7 @@ void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
void SILowerControlFlow::findMaskOperands(MachineInstr &MI, unsigned OpNo,
SmallVectorImpl<MachineOperand> &Src) const {
MachineOperand &Op = MI.getOperand(OpNo);
- if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg())) {
+ if (!Op.isReg() || !Op.getReg().isVirtual()) {
Src.push_back(Op);
return;
}
@@ -539,7 +550,7 @@ void SILowerControlFlow::findMaskOperands(MachineInstr &MI, unsigned OpNo,
for (const auto &SrcOp : Def->explicit_operands())
if (SrcOp.isReg() && SrcOp.isUse() &&
- (Register::isVirtualRegister(SrcOp.getReg()) || SrcOp.getReg() == Exec))
+ (SrcOp.getReg().isVirtual() || SrcOp.getReg() == Exec))
Src.push_back(SrcOp);
}
@@ -593,15 +604,18 @@ void SILowerControlFlow::optimizeEndCf() {
if (LIS)
LIS->RemoveMachineInstrFromMaps(*MI);
MI->eraseFromParent();
+ removeMBBifRedundant(MBB);
}
}
}
-void SILowerControlFlow::process(MachineInstr &MI) {
+MachineBasicBlock *SILowerControlFlow::process(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
MachineBasicBlock::iterator I(MI);
MachineInstr *Prev = (I != MBB.begin()) ? &*(std::prev(I)) : nullptr;
+ MachineBasicBlock *SplitBB = &MBB;
+
switch (MI.getOpcode()) {
case AMDGPU::SI_IF:
emitIf(MI);
@@ -620,7 +634,7 @@ void SILowerControlFlow::process(MachineInstr &MI) {
break;
case AMDGPU::SI_END_CF:
- emitEndCf(MI);
+ SplitBB = emitEndCf(MI);
break;
default:
@@ -645,6 +659,147 @@ void SILowerControlFlow::process(MachineInstr &MI) {
break;
}
}
+
+ return SplitBB;
+}
+
+void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB,
+ MachineInstr &MI) {
+ MachineFunction &MF = *MBB->getParent();
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ bool IsWave32 = ST.isWave32();
+
+ if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {
+ // This should be before all vector instructions.
+ BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(),
+ TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), Exec)
+ .addImm(MI.getOperand(0).getImm());
+ if (LIS)
+ LIS->RemoveMachineInstrFromMaps(MI);
+ MI.eraseFromParent();
+ return;
+ }
+
+ // Extract the thread count from an SGPR input and set EXEC accordingly.
+ // Since BFM can't shift by 64, handle that case with CMP + CMOV.
+ //
+ // S_BFE_U32 count, input, {shift, 7}
+ // S_BFM_B64 exec, count, 0
+ // S_CMP_EQ_U32 count, 64
+ // S_CMOV_B64 exec, -1
+ Register InputReg = MI.getOperand(0).getReg();
+ MachineInstr *FirstMI = &*MBB->begin();
+ if (InputReg.isVirtual()) {
+ MachineInstr *DefInstr = MRI->getVRegDef(InputReg);
+ assert(DefInstr && DefInstr->isCopy());
+ if (DefInstr->getParent() == MBB) {
+ if (DefInstr != FirstMI) {
+ // If the `InputReg` is defined in current block, we also need to
+ // move that instruction to the beginning of the block.
+ DefInstr->removeFromParent();
+ MBB->insert(FirstMI, DefInstr);
+ if (LIS)
+ LIS->handleMove(*DefInstr);
+ } else {
+ // If first instruction is definition then move pointer after it.
+ FirstMI = &*std::next(FirstMI->getIterator());
+ }
+ }
+ }
+
+ // Insert instruction sequence at block beginning (before vector operations).
+ const DebugLoc DL = MI.getDebugLoc();
+ const unsigned WavefrontSize = ST.getWavefrontSize();
+ const unsigned Mask = (WavefrontSize << 1) - 1;
+ Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass);
+ auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg)
+ .addReg(InputReg)
+ .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
+ auto BfmMI =
+ BuildMI(*MBB, FirstMI, DL,
+ TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec)
+ .addReg(CountReg)
+ .addImm(0);
+ auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
+ .addReg(CountReg, RegState::Kill)
+ .addImm(WavefrontSize);
+ auto CmovMI =
+ BuildMI(*MBB, FirstMI, DL,
+ TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+ Exec)
+ .addImm(-1);
+
+ if (!LIS) {
+ MI.eraseFromParent();
+ return;
+ }
+
+ LIS->RemoveMachineInstrFromMaps(MI);
+ MI.eraseFromParent();
+
+ LIS->InsertMachineInstrInMaps(*BfeMI);
+ LIS->InsertMachineInstrInMaps(*BfmMI);
+ LIS->InsertMachineInstrInMaps(*CmpMI);
+ LIS->InsertMachineInstrInMaps(*CmovMI);
+
+ LIS->removeInterval(InputReg);
+ LIS->createAndComputeVirtRegInterval(InputReg);
+ LIS->createAndComputeVirtRegInterval(CountReg);
+}
+
+bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
+ auto GetFallThroughSucc = [=](MachineBasicBlock *B) -> MachineBasicBlock * {
+ auto *S = B->getNextNode();
+ if (!S)
+ return nullptr;
+ if (B->isSuccessor(S)) {
+ // The only fallthrough candidate
+ MachineBasicBlock::iterator I(B->getFirstInstrTerminator());
+ MachineBasicBlock::iterator E = B->end();
+ for (; I != E; I++) {
+ if (I->isBranch() && TII->getBranchDestBlock(*I) == S)
+ // We have unoptimized branch to layout successor
+ return nullptr;
+ }
+ }
+ return S;
+ };
+
+ for (auto &I : MBB.instrs()) {
+ if (!I.isDebugInstr() && !I.isUnconditionalBranch())
+ return false;
+ }
+
+ assert(MBB.succ_size() == 1 && "MBB has more than one successor");
+
+ MachineBasicBlock *Succ = *MBB.succ_begin();
+ MachineBasicBlock *FallThrough = nullptr;
+
+ while (!MBB.predecessors().empty()) {
+ MachineBasicBlock *P = *MBB.pred_begin();
+ if (GetFallThroughSucc(P) == &MBB)
+ FallThrough = P;
+ P->ReplaceUsesOfBlockWith(&MBB, Succ);
+ }
+ MBB.removeSuccessor(Succ);
+ if (LIS) {
+ for (auto &I : MBB.instrs())
+ LIS->RemoveMachineInstrFromMaps(I);
+ }
+ MBB.clear();
+ MBB.eraseFromParent();
+ if (FallThrough && !FallThrough->isLayoutSuccessor(Succ)) {
+ if (!GetFallThroughSucc(Succ)) {
+ MachineFunction *MF = FallThrough->getParent();
+ MachineFunction::iterator FallThroughPos(FallThrough);
+ MF->splice(std::next(FallThroughPos), Succ);
+ } else
+ BuildMI(*FallThrough, FallThrough->end(),
+ FallThrough->findBranchDebugLoc(), TII->get(AMDGPU::S_BRANCH))
+ .addMBB(Succ);
+ }
+
+ return true;
}
bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
@@ -666,6 +821,7 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
MovTermOpc = AMDGPU::S_MOV_B32_term;
Andn2TermOpc = AMDGPU::S_ANDN2_B32_term;
XorTermrOpc = AMDGPU::S_XOR_B32_term;
+ OrTermrOpc = AMDGPU::S_OR_B32_term;
OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B32;
Exec = AMDGPU::EXEC_LO;
} else {
@@ -675,6 +831,7 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
MovTermOpc = AMDGPU::S_MOV_B64_term;
Andn2TermOpc = AMDGPU::S_ANDN2_B64_term;
XorTermrOpc = AMDGPU::S_XOR_B64_term;
+ OrTermrOpc = AMDGPU::S_OR_B64_term;
OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B64;
Exec = AMDGPU::EXEC;
}
@@ -682,19 +839,21 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
SmallVector<MachineInstr *, 32> Worklist;
MachineFunction::iterator NextBB;
- for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
- BI != BE; BI = NextBB) {
+ for (MachineFunction::iterator BI = MF.begin();
+ BI != MF.end(); BI = NextBB) {
NextBB = std::next(BI);
- MachineBasicBlock &MBB = *BI;
+ MachineBasicBlock *MBB = &*BI;
- MachineBasicBlock::iterator I, Next;
- for (I = MBB.begin(); I != MBB.end(); I = Next) {
+ MachineBasicBlock::iterator I, E, Next;
+ E = MBB->end();
+ for (I = MBB->begin(); I != E; I = Next) {
Next = std::next(I);
MachineInstr &MI = *I;
+ MachineBasicBlock *SplitMBB = MBB;
switch (MI.getOpcode()) {
case AMDGPU::SI_IF:
- process(MI);
+ SplitMBB = process(MI);
break;
case AMDGPU::SI_ELSE:
@@ -705,12 +864,25 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
if (InsertKillCleanups)
Worklist.push_back(&MI);
else
- process(MI);
+ SplitMBB = process(MI);
+ break;
+
+ // FIXME: find a better place for this
+ case AMDGPU::SI_INIT_EXEC:
+ case AMDGPU::SI_INIT_EXEC_FROM_INPUT:
+ lowerInitExec(MBB, MI);
+ if (LIS)
+ LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
break;
default:
break;
}
+
+ if (SplitMBB != MBB) {
+ MBB = Next->getParent();
+ E = MBB->end();
+ }
}
}
generated by cgit v1.3 (git 2.53.0) at 2026-09-05 03:02:45 +0000