summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SIInsertSkips.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/SIInsertSkips.cpp163
1 files changed, 92 insertions, 71 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
index 052db5f6ea71..9d31cd5cedc3 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
@@ -14,30 +14,11 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
-#include "AMDGPUSubtarget.h"
-#include "SIInstrInfo.h"
-#include "SIMachineFunctionInfo.h"
+#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/DepthFirstIterator.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineDominators.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/IR/CallingConv.h"
-#include "llvm/IR/DebugLoc.h"
#include "llvm/InitializePasses.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Target/TargetMachine.h"
-#include <cassert>
-#include <cstdint>
-#include <iterator>
using namespace llvm;
@@ -58,16 +39,18 @@ private:
MachineDominatorTree *MDT = nullptr;
MachineBasicBlock *EarlyExitBlock = nullptr;
+ bool EarlyExitClearsExec = false;
bool shouldSkip(const MachineBasicBlock &From,
const MachineBasicBlock &To) const;
bool dominatesAllReachable(MachineBasicBlock &MBB);
- void createEarlyExitBlock(MachineBasicBlock &MBB);
+ void ensureEarlyExitBlock(MachineBasicBlock &MBB, bool ClearExec);
void skipIfDead(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
DebugLoc DL);
bool kill(MachineInstr &MI);
+ void earlyTerm(MachineInstr &MI);
bool skipMaskBranch(MachineInstr &MI, MachineBasicBlock &MBB);
@@ -164,31 +147,62 @@ bool SIInsertSkips::dominatesAllReachable(MachineBasicBlock &MBB) {
return true;
}
-static void generatePsEndPgm(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator I, DebugLoc DL,
- const SIInstrInfo *TII) {
- // Generate "null export; s_endpgm".
- BuildMI(MBB, I, DL, TII->get(AMDGPU::EXP_DONE))
- .addImm(0x09) // V_008DFC_SQ_EXP_NULL
- .addReg(AMDGPU::VGPR0, RegState::Undef)
- .addReg(AMDGPU::VGPR0, RegState::Undef)
- .addReg(AMDGPU::VGPR0, RegState::Undef)
- .addReg(AMDGPU::VGPR0, RegState::Undef)
- .addImm(1) // vm
- .addImm(0) // compr
- .addImm(0); // en
+static void generateEndPgm(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I, DebugLoc DL,
+ const SIInstrInfo *TII, bool IsPS) {
+ // "null export"
+ if (IsPS) {
+ BuildMI(MBB, I, DL, TII->get(AMDGPU::EXP_DONE))
+ .addImm(AMDGPU::Exp::ET_NULL)
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addReg(AMDGPU::VGPR0, RegState::Undef)
+ .addImm(1) // vm
+ .addImm(0) // compr
+ .addImm(0); // en
+ }
+ // s_endpgm
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0);
}
-void SIInsertSkips::createEarlyExitBlock(MachineBasicBlock &MBB) {
+void SIInsertSkips::ensureEarlyExitBlock(MachineBasicBlock &MBB,
+ bool ClearExec) {
MachineFunction *MF = MBB.getParent();
DebugLoc DL;
- assert(!EarlyExitBlock);
- EarlyExitBlock = MF->CreateMachineBasicBlock();
- MF->insert(MF->end(), EarlyExitBlock);
+ if (!EarlyExitBlock) {
+ EarlyExitBlock = MF->CreateMachineBasicBlock();
+ MF->insert(MF->end(), EarlyExitBlock);
+ generateEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII,
+ MF->getFunction().getCallingConv() ==
+ CallingConv::AMDGPU_PS);
+ EarlyExitClearsExec = false;
+ }
- generatePsEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII);
+ if (ClearExec && !EarlyExitClearsExec) {
+ const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
+ unsigned Mov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
+ Register Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+ auto ExitI = EarlyExitBlock->getFirstNonPHI();
+ BuildMI(*EarlyExitBlock, ExitI, DL, TII->get(Mov), Exec).addImm(0);
+ EarlyExitClearsExec = true;
+ }
+}
+
+static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
+ MachineDominatorTree *MDT) {
+ MachineBasicBlock *SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/ true);
+
+ // Update dominator tree
+ using DomTreeT = DomTreeBase<MachineBasicBlock>;
+ SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
+ for (MachineBasicBlock *Succ : SplitBB->successors()) {
+ DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});
+ DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ});
+ }
+ DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB});
+ MDT->getBase().applyUpdates(DTUpdates);
}
/// Insert an "if exec=0 { null export; s_endpgm }" sequence before the given
@@ -196,6 +210,7 @@ void SIInsertSkips::createEarlyExitBlock(MachineBasicBlock &MBB) {
void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I, DebugLoc DL) {
MachineFunction *MF = MBB.getParent();
+ (void)MF;
assert(MF->getFunction().getCallingConv() == CallingConv::AMDGPU_PS);
// It is possible for an SI_KILL_*_TERMINATOR to sit at the bottom of a
@@ -211,45 +226,22 @@ void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
// In this case, we write the "null_export; s_endpgm" skip code in the
// already-existing basic block.
auto NextBBI = std::next(MBB.getIterator());
- bool NoSuccessor = I == MBB.end() &&
- llvm::find(MBB.successors(), &*NextBBI) == MBB.succ_end();
+ bool NoSuccessor =
+ I == MBB.end() && !llvm::is_contained(MBB.successors(), &*NextBBI);
if (NoSuccessor) {
- generatePsEndPgm(MBB, I, DL, TII);
+ generateEndPgm(MBB, I, DL, TII, true);
} else {
- if (!EarlyExitBlock) {
- createEarlyExitBlock(MBB);
- // Update next block pointer to reflect any new blocks
- NextBBI = std::next(MBB.getIterator());
- }
+ ensureEarlyExitBlock(MBB, false);
- auto BranchMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
- .addMBB(EarlyExitBlock);
+ MachineInstr *BranchMI =
+ BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
+ .addMBB(EarlyExitBlock);
// Split the block if the branch will not come at the end.
auto Next = std::next(BranchMI->getIterator());
- if (Next != MBB.end() && !Next->isTerminator()) {
- MachineBasicBlock *SplitBB =
- MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- MF->insert(NextBBI, SplitBB);
- SplitBB->splice(SplitBB->begin(), &MBB, I, MBB.end());
- SplitBB->transferSuccessorsAndUpdatePHIs(&MBB);
- // FIXME: the expectation is that this will be used near the beginning
- // of a block so just assume all registers are still live.
- for (auto LiveIn : MBB.liveins())
- SplitBB->addLiveIn(LiveIn);
- MBB.addSuccessor(SplitBB);
-
- // Update dominator tree
- using DomTreeT = DomTreeBase<MachineBasicBlock>;
- SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
- for (MachineBasicBlock *Succ : SplitBB->successors()) {
- DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});
- DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ});
- }
- DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB});
- MDT->getBase().applyUpdates(DTUpdates);
- }
+ if (Next != MBB.end() && !Next->isTerminator())
+ splitBlock(MBB, *BranchMI, MDT);
MBB.addSuccessor(EarlyExitBlock);
MDT->getBase().insertEdge(&MBB, EarlyExitBlock);
@@ -382,6 +374,23 @@ bool SIInsertSkips::kill(MachineInstr &MI) {
}
}
+void SIInsertSkips::earlyTerm(MachineInstr &MI) {
+ MachineBasicBlock &MBB = *MI.getParent();
+ const DebugLoc DL = MI.getDebugLoc();
+
+ ensureEarlyExitBlock(MBB, true);
+
+ auto BranchMI = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC0))
+ .addMBB(EarlyExitBlock);
+ auto Next = std::next(MI.getIterator());
+
+ if (Next != MBB.end() && !Next->isTerminator())
+ splitBlock(MBB, *BranchMI, MDT);
+
+ MBB.addSuccessor(EarlyExitBlock);
+ MDT->getBase().insertEdge(&MBB, EarlyExitBlock);
+}
+
// Returns true if a branch over the block was inserted.
bool SIInsertSkips::skipMaskBranch(MachineInstr &MI,
MachineBasicBlock &SrcMBB) {
@@ -407,6 +416,7 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
SkipThreshold = SkipThresholdFlag;
SmallVector<MachineInstr *, 4> KillInstrs;
+ SmallVector<MachineInstr *, 4> EarlyTermInstrs;
bool MadeChange = false;
for (MachineBasicBlock &MBB : MF) {
@@ -465,18 +475,29 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
}
break;
+ case AMDGPU::SI_EARLY_TERMINATE_SCC0:
+ EarlyTermInstrs.push_back(&MI);
+ break;
+
default:
break;
}
}
}
+ for (MachineInstr *Instr : EarlyTermInstrs) {
+ // Early termination in GS does nothing
+ if (MF.getFunction().getCallingConv() != CallingConv::AMDGPU_GS)
+ earlyTerm(*Instr);
+ Instr->eraseFromParent();
+ }
for (MachineInstr *Kill : KillInstrs) {
skipIfDead(*Kill->getParent(), std::next(Kill->getIterator()),
Kill->getDebugLoc());
Kill->eraseFromParent();
}
KillInstrs.clear();
+ EarlyTermInstrs.clear();
EarlyExitBlock = nullptr;
return MadeChange;
generated by cgit v1.3 (git 2.53.0) at 2026-07-28 21:08:44 +0000