diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 455 |
1 files changed, 86 insertions, 369 deletions
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 873d18e30a43..0976fccf78d8 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -14,7 +14,6 @@ #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/ScheduleDAG.h" @@ -45,9 +44,10 @@ static cl::opt<unsigned, false, MFMAPaddingRatioParser> cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")); -static cl::opt<unsigned> MaxExhaustiveHazardSearch( - "amdgpu-max-exhaustive-hazard-search", cl::init(128), cl::Hidden, - cl::desc("Maximum function size for exhausive hazard search")); +// This is intended for debugging purposes only. +static cl::opt<unsigned> + NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden, + cl::desc("Insert a s_nop x before every instruction")); //===----------------------------------------------------------------------===// // Hazard Recognizer Implementation @@ -60,7 +60,6 @@ GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), - UseVALUReadHazardExhaustiveSearch(false), ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) { MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); @@ -117,25 +116,6 @@ static bool isSMovRel(unsigned Opcode) { } } -static bool isDGEMM(unsigned Opcode) { - return AMDGPU::getMAIIsDGEMM(Opcode); -} - -static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { - unsigned Opcode = MI.getOpcode(); - - if (!SIInstrInfo::isMAI(MI) || - isDGEMM(Opcode) || - Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || - Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) - return false; - - if (!ST.hasGFX940Insts()) - return true; - - return AMDGPU::getMAIIsGFX940XDL(Opcode); -} - static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI) { if (TII.isAlwaysGDS(MI.getOpcode())) @@ -208,10 +188,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { if (ST.hasNoDataDepHazard()) return NoHazard; - // FIXME: Should flat be considered vmem? - if ((SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI)) - && checkVMEMHazards(MI) > 0) + if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0) return HazardType; if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) @@ -227,8 +204,8 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { return HazardType; if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || - SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) + SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) && + checkMAIVALUHazards(MI) > 0) return HazardType; if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) @@ -254,9 +231,8 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) return HazardType; - if ((SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI) || - SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) + if ((SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) && + checkMAILdStHazards(MI) > 0) return HazardType; if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) @@ -329,7 +305,7 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { unsigned W = PreEmitNoopsCommon(MI); fixHazards(MI); CurrCycleInstr = nullptr; - return W; + return std::max(W, NopPadding.getValue()); } unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { @@ -349,7 +325,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { if (ST.hasNoDataDepHazard()) return WaitStates; - if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) + if (SIInstrInfo::isVMEM(*MI)) WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); if (SIInstrInfo::isVALU(*MI)) @@ -365,8 +341,8 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || - SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) + SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) && + checkMAIVALUHazards(MI) > 0) WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); if (MI->isInlineAsm()) @@ -394,9 +370,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { if (SIInstrInfo::isMAI(*MI)) return std::max(WaitStates, checkMAIHazards(MI)); - if (SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI) || - SIInstrInfo::isDS(*MI)) + if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) return std::max(WaitStates, checkMAILdStHazards(MI)); if (ST.hasGFX950Insts() && isPermlane(*MI)) @@ -448,7 +422,8 @@ void GCNHazardRecognizer::AdvanceCycle() { } void GCNHazardRecognizer::RecedeCycle() { - llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); + assert(!IsHazardRecognizerMode && + "Bottom-up scheduling shouldn't run in hazard recognizer mode"); } //===----------------------------------------------------------------------===// @@ -623,7 +598,7 @@ static bool breaksSMEMSoftClause(MachineInstr *MI) { } static bool breaksVMEMSoftClause(MachineInstr *MI) { - return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); + return !SIInstrInfo::isVMEM(*MI); } int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { @@ -1217,7 +1192,6 @@ void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { fixWMMAHazards(MI); fixShift64HighRegBug(MI); fixVALUMaskWriteHazard(MI); - fixVALUReadSGPRHazard(MI); fixRequiredExportPriority(MI); } @@ -1276,8 +1250,7 @@ bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { const SIRegisterInfo *TRI = ST.getRegisterInfo(); auto IsHazardFn = [TRI, MI](const MachineInstr &I) { - if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) && - !SIInstrInfo::isFLAT(I)) + if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I)) return false; for (const MachineOperand &Def : MI->defs()) { @@ -1317,7 +1290,7 @@ bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { if (!SIInstrInfo::isVALU(*MI)) return false; - unsigned SDSTName; + AMDGPU::OpName SDSTName; switch (MI->getOpcode()) { case AMDGPU::V_READLANE_B32: case AMDGPU::V_READFIRSTLANE_B32: @@ -1451,8 +1424,8 @@ static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, for (auto &MBB : MF) { for (auto &MI : MBB) { HasLds |= SIInstrInfo::isDS(MI); - HasVmem |= - SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); + HasVmem |= (SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || + SIInstrInfo::isSegmentSpecificFLAT(MI); if (HasLds && HasVmem) return true; } @@ -1476,7 +1449,8 @@ bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { auto IsHazardInst = [](const MachineInstr &MI) { if (SIInstrInfo::isDS(MI)) return 1; - if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) + if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || + SIInstrInfo::isSegmentSpecificFLAT(MI)) return 2; return 0; }; @@ -1543,8 +1517,8 @@ bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { if (WaitStates >= NoHazardWaitStates) return true; // Instructions which cause va_vdst==0 expire hazard - return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || - SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I); + return SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || + SIInstrInfo::isEXP(I); }; auto GetWaitStatesFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) ? 1 : 0; @@ -1575,8 +1549,7 @@ bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { const Register VDSTReg = VDST->getReg(); auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) { - if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) && - !SIInstrInfo::isDS(I)) + if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I)) return false; return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); }; @@ -1661,8 +1634,8 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { return HazardExpired; // Instructions which cause va_vdst==0 expire hazard - if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || - SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || + if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || + SIInstrInfo::isEXP(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0)) return HazardExpired; @@ -1798,8 +1771,8 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { return HazardExpired; // Instructions which cause va_vdst==0 expire hazard - if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || - SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || + if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || + SIInstrInfo::isEXP(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && I.getOperand(0).getImm() == 0x0fff)) return HazardExpired; @@ -2029,7 +2002,7 @@ int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { return 0; auto IsHazardFn = [](const MachineInstr &I) { - if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I)) + if (!SIInstrInfo::isVMEM(I)) return false; return SIInstrInfo::isFPAtomic(I); }; @@ -2297,12 +2270,14 @@ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { return NumPasses + 2; } -static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { - // 2 pass -> 5 - // 4 pass -> 7 - // 8 pass -> 11 - // 16 pass -> 19 - return NumPasses + 3; +static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses, + bool IsGFX950) { + // xdl def cycles | gfx942 | gfx950 + // 2 pass | 5 5 + // 4 pass | 7 8 + // 8 pass | 11 12 + // 16 pass | 19 20 + return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { @@ -2380,7 +2355,8 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { unsigned Opc1 = MI1->getOpcode(); int NeedWaitStates = 0; if (OpNo == SrcCIdx) { - if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { + if (!SIInstrInfo::isDGEMM(Opc) && + (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) { NeedWaitStates = 0; } else if (FullReg) { if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || @@ -2397,7 +2373,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: - if (!isXDL(ST, *MI)) + if (!TII.isXDL(*MI)) NeedWaitStates = ST.hasGFX950Insts() ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates @@ -2405,18 +2381,18 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { break; case AMDGPU::V_MFMA_F64_4X4X4F64_e64: case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: - if (!isXDL(ST, *MI)) + if (!TII.isXDL(*MI)) NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; break; default: int NumPasses = TSchedModel.computeInstrLatency(MI1); if (ST.hasGFX940Insts()) { - if (isXDL(ST, *MI) && !isXDL(ST, *MI1)) + if (TII.isXDL(*MI) && !TII.isXDL(*MI1)) break; NeedWaitStates = - isXDL(ST, *MI1) - ? (isXDL(ST, *MI) + TII.isXDL(*MI1) + ? (TII.isXDL(*MI) ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates( @@ -2429,18 +2405,19 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { switch (NumPasses) { case 2: NeedWaitStates = - isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates - : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; + SIInstrInfo::isDGEMM(Opc) + ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates + : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; break; case 8: NeedWaitStates = - isDGEMM(Opc) + SIInstrInfo::isDGEMM(Opc) ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; break; case 16: NeedWaitStates = - isDGEMM(Opc) + SIInstrInfo::isDGEMM(Opc) ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; break; @@ -2469,9 +2446,9 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MI1) + TII.isXDL(*MI1) ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates( - NumPasses) + NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates( NumPasses); break; @@ -2603,20 +2580,24 @@ static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { return NumPasses + 2; } -static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { - // 2 pass -> 5 - // 4 pass -> 7 - // 8 pass -> 11 - // 16 pass -> 19 - return NumPasses + 3; +static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses, + bool IsGFX950) { + // xdl def cycles | gfx942 | gfx950 + // 2 pass | 5 5 + // 4 pass | 7 8 + // 8 pass | 11 12 + // 16 pass | 19 20 + return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } -static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { - // 2 pass -> 5 - // 4 pass -> 7 - // 8 pass -> 11 - // 16 pass -> 19 - return NumPasses + 3; +static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses, + bool IsGFX950) { + // xdl def cycles | gfx942 | gfx950 + // 2 pass | 5 5 + // 4 pass | 7 8 + // 8 pass | 11 12 + // 16 pass | 19 20 + return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { @@ -2632,7 +2613,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { return 0; auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { - return isDGEMM(MI.getOpcode()); + return SIInstrInfo::isDGEMM(MI.getOpcode()); }; // This is checked in checkMAIHazards90A() @@ -2643,9 +2624,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { int WaitStatesNeeded = 0; - bool IsMem = SIInstrInfo::isVMEM(*MI) || - SIInstrInfo::isFLAT(*MI) || - SIInstrInfo::isDS(*MI); + bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI); bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); bool IsVALU = SIInstrInfo::isVALU(*MI); @@ -2671,7 +2650,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { bool DGEMMAfterVALUWrite = false; auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { // Found DGEMM on reverse traversal to def. - if (isDGEMM(MI.getOpcode())) + if (SIInstrInfo::isDGEMM(MI.getOpcode())) DGEMMAfterVALUWrite = true; // Only hazard if register is defined by a VALU and a DGEMM is found after @@ -2746,7 +2725,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { int NumPasses = HazardDefLatency; int NeedWaitStates = MaxWaitStates; - if (isDGEMM(MFMA->getOpcode())) { + if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (HazardDefLatency) { case 4: NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates @@ -2766,8 +2745,9 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } } else if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MFMA) - ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses) + TII.isXDL(*MFMA) + ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates( + NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( NumPasses); } else { @@ -2838,7 +2818,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { int NeedWaitStates = MaxWaitStates; int NumPasses = TSchedModel.computeInstrLatency(MFMA); - if (isDGEMM(MFMA->getOpcode())) { + if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (NumPasses) { case 4: NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; @@ -2852,8 +2832,9 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } } else if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MFMA) - ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses) + TII.isXDL(*MFMA) + ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates( + NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses); } else { switch (NumPasses) { @@ -2879,11 +2860,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { - if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || + if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) || !MI.readsRegister(Reg, &TRI)) return false; - if (ST.hasGFX940Insts() && !isXDL(ST, MI)) + if (ST.hasGFX940Insts() && !TII.isXDL(MI)) return false; const MachineOperand *SrcC = @@ -3010,7 +2991,9 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { switch (I.getOpcode()) { case AMDGPU::V_ADDC_U32_e32: case AMDGPU::V_ADDC_U32_dpp: + case AMDGPU::V_CNDMASK_B16_t16_e32: case AMDGPU::V_CNDMASK_B16_fake16_e32: + case AMDGPU::V_CNDMASK_B16_t16_dpp: case AMDGPU::V_CNDMASK_B16_fake16_dpp: case AMDGPU::V_CNDMASK_B32_e32: case AMDGPU::V_CNDMASK_B32_dpp: @@ -3026,7 +3009,9 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { HazardReg == AMDGPU::VCC_HI; case AMDGPU::V_ADDC_U32_e64: case AMDGPU::V_ADDC_U32_e64_dpp: + case AMDGPU::V_CNDMASK_B16_t16_e64: case AMDGPU::V_CNDMASK_B16_fake16_e64: + case AMDGPU::V_CNDMASK_B16_t16_e64_dpp: case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp: case AMDGPU::V_CNDMASK_B32_e64: case AMDGPU::V_CNDMASK_B32_e64_dpp: @@ -3106,274 +3091,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { return true; } -// Return the numeric ID 0-63 of an 64b SGPR pair for a given SGPR. -// i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc -static std::optional<unsigned> sgprPairNumber(Register Reg, - const SIRegisterInfo &TRI) { - switch (Reg) { - case AMDGPU::M0: - case AMDGPU::EXEC: - case AMDGPU::EXEC_LO: - case AMDGPU::EXEC_HI: - case AMDGPU::SGPR_NULL: - case AMDGPU::SGPR_NULL64: - return {}; - default: - break; - } - unsigned RegN = TRI.getEncodingValue(Reg); - if (RegN > 127) - return {}; - return (RegN >> 1) & 0x3f; -} - -// For VALUReadSGPRHazard: pre-compute a bit vector of all SGPRs used by VALUs. -void GCNHazardRecognizer::computeVALUHazardSGPRs(MachineFunction *MMF) { - assert(MMF == &MF); - - // Assume non-empty vector means it has already been computed. - if (!VALUReadHazardSGPRs.empty()) - return; - - auto CallingConv = MF.getFunction().getCallingConv(); - bool IsCallFree = - AMDGPU::isEntryFunctionCC(CallingConv) && !MF.getFrameInfo().hasCalls(); - - // Exhaustive search is only viable in non-caller/callee functions where - // VALUs will be exposed to the hazard recognizer. - UseVALUReadHazardExhaustiveSearch = - IsCallFree && MF.getTarget().getOptLevel() > CodeGenOptLevel::None && - MF.getInstructionCount() <= MaxExhaustiveHazardSearch; - - // Consider all SGPRs hazards if the shader uses function calls or is callee. - bool UseVALUUseCache = - IsCallFree && MF.getTarget().getOptLevel() > CodeGenOptLevel::None; - VALUReadHazardSGPRs.resize(64, !UseVALUUseCache); - if (!UseVALUUseCache) - return; - - // Perform a post ordered reverse scan to find VALUs which read an SGPR - // before a SALU write to the same SGPR. This provides a reduction in - // hazard insertion when all VALU access to an SGPR occurs after its last - // SALU write, when compared to a linear scan. - const MachineRegisterInfo &MRI = MF.getRegInfo(); - BitVector SALUWriteSGPRs(64), ReadSGPRs(64); - MachineCycleInfo CI; - CI.compute(*MMF); - - for (auto *MBB : post_order(&MF)) { - bool InCycle = CI.getCycle(MBB) != nullptr; - for (auto &MI : reverse(MBB->instrs())) { - bool IsVALU = SIInstrInfo::isVALU(MI); - bool IsSALU = SIInstrInfo::isSALU(MI); - if (!IsVALU && !IsSALU) - continue; - - for (const MachineOperand &Op : MI.operands()) { - if (!Op.isReg()) - continue; - Register Reg = Op.getReg(); - assert(!Op.getSubReg()); - // Only consider implicit operands of VCC. - if (Op.isImplicit() && !(Reg == AMDGPU::VCC_LO || - Reg == AMDGPU::VCC_HI || Reg == AMDGPU::VCC)) - continue; - if (!TRI.isSGPRReg(MRI, Reg)) - continue; - auto RegN = sgprPairNumber(Reg, TRI); - if (!RegN) - continue; - if (IsVALU && Op.isUse()) { - // Note: any access within a cycle must be considered a hazard. - if (InCycle || (ReadSGPRs[*RegN] && SALUWriteSGPRs[*RegN])) - VALUReadHazardSGPRs.set(*RegN); - ReadSGPRs.set(*RegN); - } else if (IsSALU) { - if (Op.isDef()) - SALUWriteSGPRs.set(*RegN); - else - ReadSGPRs.set(*RegN); - } - } - } - } -} - -bool GCNHazardRecognizer::fixVALUReadSGPRHazard(MachineInstr *MI) { - if (!ST.hasVALUReadSGPRHazard()) - return false; - - // The hazard sequence is fundamentally three instructions: - // 1. VALU reads SGPR - // 2. SALU writes SGPR - // 3. VALU/SALU reads SGPR - // Try to avoid searching for (1) because the expiry point of the hazard is - // indeterminate; however, the hazard between (2) and (3) can expire if the - // gap contains sufficient SALU instructions with no usage of SGPR from (1). - // Note: SGPRs must be considered as 64-bit pairs as hazard exists - // even if individual SGPRs are accessed. - - bool MIIsSALU = SIInstrInfo::isSALU(*MI); - bool MIIsVALU = SIInstrInfo::isVALU(*MI); - if (!(MIIsSALU || MIIsVALU)) - return false; - - // Avoid expensive search when compile time is priority by - // mitigating every SALU which writes an SGPR. - if (MF.getTarget().getOptLevel() == CodeGenOptLevel::None) { - if (!SIInstrInfo::isSALU(*MI) || SIInstrInfo::isSOPP(*MI)) - return false; - - const MachineOperand *SDSTOp = - TII.getNamedOperand(*MI, AMDGPU::OpName::sdst); - if (!SDSTOp || !SDSTOp->isReg()) - return false; - - const Register HazardReg = SDSTOp->getReg(); - if (HazardReg == AMDGPU::EXEC || HazardReg == AMDGPU::EXEC_LO || - HazardReg == AMDGPU::EXEC_HI || HazardReg == AMDGPU::M0) - return false; - - // Add s_wait_alu sa_sdst(0) after SALU write. - auto NextMI = std::next(MI->getIterator()); - auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(), - TII.get(AMDGPU::S_WAITCNT_DEPCTR)) - .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); - - // SALU write may be s_getpc in a bundle. - updateGetPCBundle(NewMI); - - return true; - } - - // Pre-compute set of SGPR pairs read by VALUs. - // Note: pass mutable pointer to MachineFunction for CycleInfo. - computeVALUHazardSGPRs(MI->getMF()); - - // If no VALUs hazard SGPRs exist then nothing to do. - if (VALUReadHazardSGPRs.none()) - return false; - - // All SGPR writes before a call/return must be flushed as the callee/caller - // will not will not see the hazard chain, i.e. (2) to (3) described above. - const bool IsSetPC = (MI->isCall() || MI->isReturn()) && - !(MI->getOpcode() == AMDGPU::S_ENDPGM || - MI->getOpcode() == AMDGPU::S_ENDPGM_SAVED); - - // Collect all SGPR sources for MI which are read by a VALU. - const MachineRegisterInfo &MRI = MF.getRegInfo(); - SmallSet<Register, 4> SGPRsUsed; - - if (!IsSetPC) { - for (const MachineOperand &Op : MI->all_uses()) { - Register OpReg = Op.getReg(); - - // Only consider VCC implicit uses on VALUs. - // The only expected SALU implicit access is SCC which is no hazard. - if (MIIsSALU && Op.isImplicit()) - continue; - - if (!TRI.isSGPRReg(MRI, OpReg)) - continue; - - auto RegN = sgprPairNumber(OpReg, TRI); - if (!RegN) - continue; - - if (!VALUReadHazardSGPRs[*RegN]) - continue; - - SGPRsUsed.insert(OpReg); - } - - // No SGPRs -> nothing to do. - if (SGPRsUsed.empty()) - return false; - } - - // A hazard is any SALU which writes one of the SGPRs read by MI. - auto IsHazardFn = [this, IsSetPC, &SGPRsUsed](const MachineInstr &I) { - if (!SIInstrInfo::isSALU(I)) - return false; - // Ensure SGPR flush before call/return by conservatively assuming every - // SALU writes an SGPR. - if (IsSetPC && I.getNumDefs() > 0) - return true; - // Check for any register writes. - return any_of(SGPRsUsed, [this, &I](Register Reg) { - return I.modifiesRegister(Reg, &TRI); - }); - }; - - const int SALUExpiryCount = SIInstrInfo::isSALU(*MI) ? 10 : 11; - auto IsExpiredFn = [&](const MachineInstr &I, int Count) { - if (Count >= SALUExpiryCount) - return true; - // s_wait_alu sa_sdst(0) on path mitigates hazard. - if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && - AMDGPU::DepCtr::decodeFieldSaSdst(I.getOperand(0).getImm()) == 0) - return true; - return false; - }; - - auto WaitStatesFn = [this, &SGPRsUsed](const MachineInstr &I) { - // Only count true SALUs as wait states. - if (!SIInstrInfo::isSALU(I) || SIInstrInfo::isSOPP(I)) - return 0; - // SALU must be unrelated to any hazard registers. - if (any_of(SGPRsUsed, - [this, &I](Register Reg) { return I.readsRegister(Reg, &TRI); })) - return 0; - return 1; - }; - - // Check for the hazard. - DenseSet<const MachineBasicBlock *> Visited; - int WaitStates = ::getWaitStatesSince(IsHazardFn, MI->getParent(), - std::next(MI->getReverseIterator()), 0, - IsExpiredFn, Visited, WaitStatesFn); - - if (WaitStates >= SALUExpiryCount) - return false; - - // Validate hazard through an exhaustive search. - if (UseVALUReadHazardExhaustiveSearch) { - // A hazard is any VALU which reads one of the paired SGPRs read by MI. - // This is searching for (1) in the hazard description. - auto hazardPair = [this](Register Reg) { - if (Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI) - return Register(AMDGPU::VCC); - auto RegN = sgprPairNumber(Reg, TRI); - return Register(AMDGPU::SGPR0_SGPR1 + *RegN); - }; - auto SearchHazardFn = [this, hazardPair, - &SGPRsUsed](const MachineInstr &I) { - if (!SIInstrInfo::isVALU(I)) - return false; - // Check for any register reads. - return any_of(SGPRsUsed, [this, hazardPair, &I](Register Reg) { - return I.readsRegister(hazardPair(Reg), &TRI); - }); - }; - auto SearchExpiredFn = [&](const MachineInstr &I, int Count) { - return false; - }; - if (::getWaitStatesSince(SearchHazardFn, MI, SearchExpiredFn) == - std::numeric_limits<int>::max()) - return false; - } - - // Add s_wait_alu sa_sdst(0) before SALU read. - auto NewMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), - TII.get(AMDGPU::S_WAITCNT_DEPCTR)) - .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); - - // SALU read may be after s_getpc in a bundle. - updateGetPCBundle(NewMI); - - return true; -} - static bool ensureEntrySetPrio(MachineFunction *MF, int Priority, const SIInstrInfo &TII) { MachineBasicBlock &EntryMBB = MF->front(); |