summaryrefslogtreecommitdiff
path: root/lib/Target/AMDGPU/SIInsertWaits.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Target/AMDGPU/SIInsertWaits.cpp')
-rw-r--r--lib/Target/AMDGPU/SIInsertWaits.cpp127
1 files changed, 94 insertions, 33 deletions
diff --git a/lib/Target/AMDGPU/SIInsertWaits.cpp b/lib/Target/AMDGPU/SIInsertWaits.cpp
index d24588d6c143..202a1e9ed8ac 100644
--- a/lib/Target/AMDGPU/SIInsertWaits.cpp
+++ b/lib/Target/AMDGPU/SIInsertWaits.cpp
@@ -21,6 +21,7 @@
#include "SIDefines.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -29,6 +30,7 @@
#define DEBUG_TYPE "si-insert-waits"
using namespace llvm;
+using namespace llvm::AMDGPU;
namespace {
@@ -59,13 +61,14 @@ private:
const SIInstrInfo *TII;
const SIRegisterInfo *TRI;
const MachineRegisterInfo *MRI;
-
- /// \brief Constant hardware limits
- static const Counters WaitCounts;
+ IsaVersion IV;
/// \brief Constant zero value
static const Counters ZeroCounts;
+ /// \brief Hardware limits
+ Counters HardwareLimits;
+
/// \brief Counter values we have already waited on.
Counters WaitedOn;
@@ -90,6 +93,9 @@ private:
bool LastInstWritesM0;
+ /// Whether or not we have flat operations outstanding.
+ bool IsFlatOutstanding;
+
/// \brief Whether the machine function returns void
bool ReturnsVoid;
@@ -145,7 +151,7 @@ public:
bool runOnMachineFunction(MachineFunction &MF) override;
- const char *getPassName() const override {
+ StringRef getPassName() const override {
return "SI insert wait instructions";
}
@@ -170,11 +176,12 @@ FunctionPass *llvm::createSIInsertWaitsPass() {
return new SIInsertWaits();
}
-const Counters SIInsertWaits::WaitCounts = { { 15, 7, 15 } };
const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
-static bool readsVCCZ(unsigned Opcode) {
- return Opcode == AMDGPU::S_CBRANCH_VCCNZ || Opcode == AMDGPU::S_CBRANCH_VCCZ;
+static bool readsVCCZ(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
+ !MI.getOperand(1).isUndef();
}
bool SIInsertWaits::hasOutstandingLGKM() const {
@@ -188,8 +195,7 @@ Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);
// Only consider stores or EXP for EXP_CNT
- Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT &&
- (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore()));
+ Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT) && MI.mayStore();
// LGKM may uses larger values
if (TSFlags & SIInstrFlags::LGKM_CNT) {
@@ -231,9 +237,10 @@ bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
if (Op.isDef())
return true;
- // For exports all registers are relevant
+ // For exports all registers are relevant.
+ // TODO: Skip undef/disabled registers.
MachineInstr &MI = *Op.getParent();
- if (MI.getOpcode() == AMDGPU::EXP)
+ if (TII->isEXP(MI))
return true;
// For stores the stored value is also relevant
@@ -245,12 +252,6 @@ bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
// operand comes before the value operand and it may have
// multiple data operands.
- if (TII->isDS(MI) || TII->isFLAT(MI)) {
- MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::data);
- if (Data && Op.isIdenticalTo(*Data))
- return true;
- }
-
if (TII->isDS(MI)) {
MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
if (Data0 && Op.isIdenticalTo(*Data0))
@@ -260,6 +261,12 @@ bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
return Data1 && Op.isIdenticalTo(*Data1);
}
+ if (TII->isFLAT(MI)) {
+ MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::vdata);
+ if (Data && Op.isIdenticalTo(*Data))
+ return true;
+ }
+
// NOTE: This assumes that the value operand is before the
// address operand, and that there is only one value operand.
for (MachineInstr::mop_iterator I = MI.operands_begin(),
@@ -292,6 +299,9 @@ void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB,
Counters Limit = ZeroCounts;
unsigned Sum = 0;
+ if (TII->mayAccessFlatAddressSpace(*I))
+ IsFlatOutstanding = true;
+
for (unsigned i = 0; i < 3; ++i) {
LastIssued.Array[i] += Increment.Array[i];
if (Increment.Array[i])
@@ -330,7 +340,7 @@ void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB,
// Remember which export instructions we have seen
if (Increment.Named.EXP) {
- ExpInstrTypesSeen |= I->getOpcode() == AMDGPU::EXP ? 1 : 2;
+ ExpInstrTypesSeen |= TII->isEXP(*I) ? 1 : 2;
}
for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
@@ -366,8 +376,9 @@ bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
// Figure out if the async instructions execute in order
bool Ordered[3];
- // VM_CNT is always ordered
- Ordered[0] = true;
+ // VM_CNT is always ordered except when there are flat instructions, which
+ // can return out of order.
+ Ordered[0] = !IsFlatOutstanding;
// EXP_CNT is unordered if we have both EXP & VM-writes
Ordered[1] = ExpInstrTypesSeen == 3;
@@ -376,7 +387,7 @@ bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
Ordered[2] = false;
// The values we are going to put into the S_WAITCNT instruction
- Counters Counts = WaitCounts;
+ Counters Counts = HardwareLimits;
// Do we really need to wait?
bool NeedWait = false;
@@ -392,7 +403,7 @@ bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
unsigned Value = LastIssued.Array[i] - Required.Array[i];
// Adjust the value to the real hardware possibilities.
- Counts.Array[i] = std::min(Value, WaitCounts.Array[i]);
+ Counts.Array[i] = std::min(Value, HardwareLimits.Array[i]);
} else
Counts.Array[i] = 0;
@@ -410,12 +421,14 @@ bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
// Build the wait instruction
BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
- .addImm((Counts.Named.VM & 0xF) |
- ((Counts.Named.EXP & 0x7) << 4) |
- ((Counts.Named.LGKM & 0xF) << 8));
+ .addImm(encodeWaitcnt(IV,
+ Counts.Named.VM,
+ Counts.Named.EXP,
+ Counts.Named.LGKM));
LastOpcodeType = OTHER;
LastInstWritesM0 = false;
+ IsFlatOutstanding = false;
return true;
}
@@ -440,9 +453,9 @@ void SIInsertWaits::handleExistingWait(MachineBasicBlock::iterator I) {
unsigned Imm = I->getOperand(0).getImm();
Counters Counts, WaitOn;
- Counts.Named.VM = Imm & 0xF;
- Counts.Named.EXP = (Imm >> 4) & 0x7;
- Counts.Named.LGKM = (Imm >> 8) & 0xF;
+ Counts.Named.VM = decodeVmcnt(IV, Imm);
+ Counts.Named.EXP = decodeExpcnt(IV, Imm);
+ Counts.Named.LGKM = decodeLgkmcnt(IV, Imm);
for (unsigned i = 0; i < 3; ++i) {
if (Counts.Array[i] <= LastIssued.Array[i])
@@ -518,26 +531,40 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
TII = ST->getInstrInfo();
TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo();
+ IV = getIsaVersion(ST->getFeatureBits());
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+ HardwareLimits.Named.VM = getVmcntBitMask(IV);
+ HardwareLimits.Named.EXP = getExpcntBitMask(IV);
+ HardwareLimits.Named.LGKM = getLgkmcntBitMask(IV);
WaitedOn = ZeroCounts;
DelayedWaitOn = ZeroCounts;
LastIssued = ZeroCounts;
LastOpcodeType = OTHER;
LastInstWritesM0 = false;
- ReturnsVoid = MF.getInfo<SIMachineFunctionInfo>()->returnsVoid();
+ IsFlatOutstanding = false;
+ ReturnsVoid = MFI->returnsVoid();
memset(&UsedRegs, 0, sizeof(UsedRegs));
memset(&DefinedRegs, 0, sizeof(DefinedRegs));
SmallVector<MachineInstr *, 4> RemoveMI;
+ SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
+
+ bool HaveScalarStores = false;
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
BI != BE; ++BI) {
MachineBasicBlock &MBB = *BI;
+
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
I != E; ++I) {
+ if (!HaveScalarStores && TII->isScalarStore(*I))
+ HaveScalarStores = true;
+
if (ST->getGeneration() <= SISubtarget::SEA_ISLANDS) {
// There is a hardware bug on CI/SI where SMRD instruction may corrupt
// vccz bit, so when we detect that an instruction may read from a
@@ -557,7 +584,7 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
}
// Check if we need to apply the bug work-around
- if (readsVCCZ(I->getOpcode()) && VCCZCorrupt) {
+ if (VCCZCorrupt && readsVCCZ(*I)) {
DEBUG(dbgs() << "Inserting vccz bug work-around before: " << *I << '\n');
// Wait on everything, not just LGKM. vccz reads usually come from
@@ -572,7 +599,7 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
// vcc and then writing it back to the register.
BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
AMDGPU::VCC)
- .addReg(AMDGPU::VCC);
+ .addReg(AMDGPU::VCC);
}
}
@@ -590,8 +617,9 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
// S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish,
// but we also want to wait for any other outstanding transfers before
// signalling other hardware blocks
- if (I->getOpcode() == AMDGPU::S_BARRIER ||
- I->getOpcode() == AMDGPU::S_SENDMSG)
+ if ((I->getOpcode() == AMDGPU::S_BARRIER &&
+ ST->needWaitcntBeforeBarrier()) ||
+ I->getOpcode() == AMDGPU::S_SENDMSG)
Required = LastIssued;
else
Required = handleOperands(*I);
@@ -605,12 +633,45 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
pushInstruction(MBB, I, Increment);
handleSendMsg(MBB, I);
+
+ if (I->getOpcode() == AMDGPU::S_ENDPGM ||
+ I->getOpcode() == AMDGPU::SI_RETURN)
+ EndPgmBlocks.push_back(&MBB);
}
// Wait for everything at the end of the MBB
Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);
}
+ if (HaveScalarStores) {
+ // If scalar writes are used, the cache must be flushed or else the next
+ // wave to reuse the same scratch memory can be clobbered.
+ //
+ // Insert s_dcache_wb at wave termination points if there were any scalar
+ // stores, and only if the cache hasn't already been flushed. This could be
+ // improved by looking across blocks for flushes in postdominating blocks
+ // from the stores but an explicitly requested flush is probably very rare.
+ for (MachineBasicBlock *MBB : EndPgmBlocks) {
+ bool SeenDCacheWB = false;
+
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I) {
+
+ if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
+ SeenDCacheWB = true;
+ else if (TII->isScalarStore(*I))
+ SeenDCacheWB = false;
+
+ // FIXME: It would be better to insert this before a waitcnt if any.
+ if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
+ I->getOpcode() == AMDGPU::SI_RETURN) && !SeenDCacheWB) {
+ Changes = true;
+ BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
+ }
+ }
+ }
+ }
+
for (MachineInstr *I : RemoveMI)
I->eraseFromParent();
generated by cgit v1.3 (git 2.53.0) at 2026-07-29 05:30:46 +0000