diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp | 1019 |
1 files changed, 782 insertions, 237 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index 0640e24b37ec..38548eaf9478 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -7,14 +7,17 @@ //===----------------------------------------------------------------------===// // /// \file -/// This pass adds instructions to enable whole quad mode for pixel -/// shaders, and whole wavefront mode for all programs. +/// This pass adds instructions to enable whole quad mode (strict or non-strict) +/// for pixel shaders, and strict whole wavefront mode for all programs. +/// +/// The "strict" prefix indicates that inactive lanes do not take part in +/// control flow, specifically an inactive lane enabled by a strict WQM/WWM will +/// always be enabled irrespective of control flow decisions. Conversely in +/// non-strict WQM inactive lanes may control flow decisions. /// /// Whole quad mode is required for derivative computations, but it interferes -/// with shader side effects (stores and atomics). This pass is run on the -/// scheduled machine IR but before register coalescing, so that machine SSA is -/// available for analysis. It ensures that WQM is enabled when necessary, but -/// disabled around stores and atomics. +/// with shader side effects (stores and atomics). It ensures that WQM is +/// enabled when necessary, but disabled around stores and atomics. /// /// When necessary, this pass creates a function prolog /// @@ -28,12 +31,21 @@ /// ... /// S_MOV_B64 EXEC, Tmp /// -/// We also compute when a sequence of instructions requires Whole Wavefront -/// Mode (WWM) and insert instructions to save and restore it: +/// We also compute when a sequence of instructions requires strict whole +/// wavefront mode (StrictWWM) and insert instructions to save and restore it: +/// +/// S_OR_SAVEEXEC_B64 Tmp, -1 +/// ... +/// S_MOV_B64 EXEC, Tmp +/// +/// When a sequence of instructions requires strict whole quad mode (StrictWQM) +/// we use a similar save and restore mechanism and force whole quad mode for +/// those instructions: /// -/// S_OR_SAVEEXEC_B64 Tmp, -1 -/// ... -/// S_MOV_B64 EXEC, Tmp +/// S_MOV_B64 Tmp, EXEC +/// S_WQM_B64 EXEC, EXEC +/// ... +/// S_MOV_B64 EXEC, Tmp /// /// In order to avoid excessive switching during sequences of Exact /// instructions, the pass first analyzes which instructions must be run in WQM @@ -62,8 +74,10 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/IR/CallingConv.h" #include "llvm/InitializePasses.h" #include "llvm/Support/raw_ostream.h" @@ -76,8 +90,10 @@ namespace { enum { StateWQM = 0x1, - StateWWM = 0x2, - StateExact = 0x4, + StateStrictWWM = 0x2, + StateStrictWQM = 0x4, + StateExact = 0x8, + StateStrict = StateStrictWWM | StateStrictWQM, }; struct PrintState { @@ -89,19 +105,23 @@ public: #ifndef NDEBUG static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) { - if (PS.State & StateWQM) - OS << "WQM"; - if (PS.State & StateWWM) { - if (PS.State & StateWQM) - OS << '|'; - OS << "WWM"; - } - if (PS.State & StateExact) { - if (PS.State & (StateWQM | StateWWM)) - OS << '|'; - OS << "Exact"; - } + static const std::pair<char, const char *> Mapping[] = { + std::make_pair(StateWQM, "WQM"), + std::make_pair(StateStrictWWM, "StrictWWM"), + std::make_pair(StateStrictWQM, "StrictWQM"), + std::make_pair(StateExact, "Exact")}; + char State = PS.State; + for (auto M : Mapping) { + if (State & M.first) { + OS << M.second; + State &= ~M.first; + + if (State) + OS << '|'; + } + } + assert(State == 0); return OS; } #endif @@ -116,6 +136,8 @@ struct BlockInfo { char Needs = 0; char InNeeds = 0; char OutNeeds = 0; + char InitialState = 0; + bool NeedsLowering = false; }; struct WorkItem { @@ -129,23 +151,33 @@ struct WorkItem { class SIWholeQuadMode : public MachineFunctionPass { private: - CallingConv::ID CallingConv; const SIInstrInfo *TII; const SIRegisterInfo *TRI; const GCNSubtarget *ST; MachineRegisterInfo *MRI; LiveIntervals *LIS; + MachineDominatorTree *MDT; + MachinePostDominatorTree *PDT; unsigned AndOpc; - unsigned XorTermrOpc; + unsigned AndN2Opc; + unsigned XorOpc; + unsigned AndSaveExecOpc; unsigned OrSaveExecOpc; - unsigned Exec; + unsigned WQMOpc; + Register Exec; + Register LiveMaskReg; DenseMap<const MachineInstr *, InstrInfo> Instructions; MapVector<MachineBasicBlock *, BlockInfo> Blocks; - SmallVector<MachineInstr *, 1> LiveMaskQueries; + + // Tracks state (WQM/StrictWWM/StrictWQM/Exact) after a given instruction + DenseMap<const MachineInstr *, char> StateTransition; + + SmallVector<MachineInstr *, 2> LiveMaskQueries; SmallVector<MachineInstr *, 4> LowerToMovInstrs; SmallVector<MachineInstr *, 4> LowerToCopyInstrs; + SmallVector<MachineInstr *, 4> KillInstrs; void printInfo(); @@ -153,6 +185,8 @@ private: std::vector<WorkItem> &Worklist); void markDefs(const MachineInstr &UseMI, LiveRange &LR, Register Reg, unsigned SubReg, char Flag, std::vector<WorkItem> &Worklist); + void markOperand(const MachineInstr &MI, const MachineOperand &Op, char Flag, + std::vector<WorkItem> &Worklist); void markInstructionUses(const MachineInstr &MI, char Flag, std::vector<WorkItem> &Worklist); char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist); @@ -167,17 +201,27 @@ private: MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC); void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SaveWQM, unsigned LiveMaskReg); + Register SaveWQM); void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SavedWQM); - void toWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SaveOrig); - void fromWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SavedOrig); - void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry); + Register SavedWQM); + void toStrictMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, + Register SaveOrig, char StrictStateNeeded); + void fromStrictMode(MachineBasicBlock &MBB, + MachineBasicBlock::iterator Before, Register SavedOrig, + char NonStrictState, char CurrentStrictState); - void lowerLiveMaskQueries(unsigned LiveMaskReg); + MachineBasicBlock *splitBlock(MachineBasicBlock *BB, MachineInstr *TermMI); + + MachineInstr *lowerKillI1(MachineBasicBlock &MBB, MachineInstr &MI, + bool IsWQM); + MachineInstr *lowerKillF32(MachineBasicBlock &MBB, MachineInstr &MI); + + void lowerBlock(MachineBasicBlock &MBB); + void processBlock(MachineBasicBlock &MBB, bool IsEntry); + + void lowerLiveMaskQueries(); void lowerCopyInstrs(); + void lowerKillInstrs(bool IsWQM); public: static char ID; @@ -193,9 +237,17 @@ public: AU.addRequired<LiveIntervals>(); AU.addPreserved<SlotIndexes>(); AU.addPreserved<LiveIntervals>(); - AU.setPreservesCFG(); + AU.addRequired<MachineDominatorTree>(); + AU.addPreserved<MachineDominatorTree>(); + AU.addRequired<MachinePostDominatorTree>(); + AU.addPreserved<MachinePostDominatorTree>(); MachineFunctionPass::getAnalysisUsage(AU); } + + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } }; } // end anonymous namespace @@ -205,6 +257,8 @@ char SIWholeQuadMode::ID = 0; INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false, false) INITIALIZE_PASS_DEPENDENCY(LiveIntervals) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree) INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false, false) @@ -241,8 +295,6 @@ void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag, assert(!(Flag & StateExact) && Flag != 0); - LLVM_DEBUG(dbgs() << "markInstruction " << PrintState(Flag) << ": " << MI); - // Remove any disabled states from the flag. The user that required it gets // an undefined value in the helper lanes. For example, this can happen if // the result of an atomic is used by instruction that requires WQM, where @@ -254,6 +306,7 @@ void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag, if ((II.Needs & Flag) == Flag) return; + LLVM_DEBUG(dbgs() << "markInstruction " << PrintState(Flag) << ": " << MI); II.Needs |= Flag; Worklist.push_back(&MI); } @@ -262,108 +315,167 @@ void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag, void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR, Register Reg, unsigned SubReg, char Flag, std::vector<WorkItem> &Worklist) { - assert(!MRI->isSSA()); - LLVM_DEBUG(dbgs() << "markDefs " << PrintState(Flag) << ": " << UseMI); LiveQueryResult UseLRQ = LR.Query(LIS->getInstructionIndex(UseMI)); - if (!UseLRQ.valueIn()) + const VNInfo *Value = UseLRQ.valueIn(); + if (!Value) return; - SmallPtrSet<const VNInfo *, 4> Visited; - SmallVector<const VNInfo *, 4> ToProcess; - ToProcess.push_back(UseLRQ.valueIn()); + // Note: this code assumes that lane masks on AMDGPU completely + // cover registers. + const LaneBitmask UseLanes = + SubReg ? TRI->getSubRegIndexLaneMask(SubReg) + : (Reg.isVirtual() ? MRI->getMaxLaneMaskForVReg(Reg) + : LaneBitmask::getNone()); + + // Perform a depth-first iteration of the LiveRange graph marking defs. + // Stop processing of a given branch when all use lanes have been defined. + // The first definition stops processing for a physical register. + struct PhiEntry { + const VNInfo *Phi; + unsigned PredIdx; + LaneBitmask DefinedLanes; + + PhiEntry(const VNInfo *Phi, unsigned PredIdx, LaneBitmask DefinedLanes) + : Phi(Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {} + }; + using VisitKey = std::pair<const VNInfo *, LaneBitmask>; + SmallVector<PhiEntry, 2> PhiStack; + SmallSet<VisitKey, 4> Visited; + LaneBitmask DefinedLanes; + unsigned NextPredIdx = 0; // Only used for processing phi nodes do { - const VNInfo *Value = ToProcess.pop_back_val(); - Visited.insert(Value); + const VNInfo *NextValue = nullptr; + const VisitKey Key(Value, DefinedLanes); + + if (!Visited.count(Key)) { + Visited.insert(Key); + // On first visit to a phi then start processing first predecessor + NextPredIdx = 0; + } if (Value->isPHIDef()) { - // Need to mark all defs used in the PHI node + // Each predecessor node in the phi must be processed as a subgraph const MachineBasicBlock *MBB = LIS->getMBBFromIndex(Value->def); assert(MBB && "Phi-def has no defining MBB"); - for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); - PI != PE; ++PI) { + + // Find next predecessor to process + unsigned Idx = NextPredIdx; + auto PI = MBB->pred_begin() + Idx; + auto PE = MBB->pred_end(); + for (; PI != PE && !NextValue; ++PI, ++Idx) { if (const VNInfo *VN = LR.getVNInfoBefore(LIS->getMBBEndIdx(*PI))) { - if (!Visited.count(VN)) - ToProcess.push_back(VN); + if (!Visited.count(VisitKey(VN, DefinedLanes))) + NextValue = VN; } } + + // If there are more predecessors to process; add phi to stack + if (PI != PE) + PhiStack.emplace_back(Value, Idx, DefinedLanes); } else { MachineInstr *MI = LIS->getInstructionFromIndex(Value->def); assert(MI && "Def has no defining instruction"); - markInstruction(*MI, Flag, Worklist); - // Iterate over all operands to find relevant definitions - for (const MachineOperand &Op : MI->operands()) { - if (!(Op.isReg() && Op.getReg() == Reg)) - continue; + if (Reg.isVirtual()) { + // Iterate over all operands to find relevant definitions + bool HasDef = false; + for (const MachineOperand &Op : MI->operands()) { + if (!(Op.isReg() && Op.isDef() && Op.getReg() == Reg)) + continue; - // Does this def cover whole register? - bool DefinesFullReg = - Op.isUndef() || !Op.getSubReg() || Op.getSubReg() == SubReg; - if (!DefinesFullReg) { - // Partial definition; need to follow and mark input value + // Compute lanes defined and overlap with use + LaneBitmask OpLanes = + Op.isUndef() ? LaneBitmask::getAll() + : TRI->getSubRegIndexLaneMask(Op.getSubReg()); + LaneBitmask Overlap = (UseLanes & OpLanes); + + // Record if this instruction defined any of use + HasDef |= Overlap.any(); + + // Mark any lanes defined + DefinedLanes |= OpLanes; + } + + // Check if all lanes of use have been defined + if ((DefinedLanes & UseLanes) != UseLanes) { + // Definition not complete; need to process input value LiveQueryResult LRQ = LR.Query(LIS->getInstructionIndex(*MI)); if (const VNInfo *VN = LRQ.valueIn()) { - if (!Visited.count(VN)) - ToProcess.push_back(VN); + if (!Visited.count(VisitKey(VN, DefinedLanes))) + NextValue = VN; } } + + // Only mark the instruction if it defines some part of the use + if (HasDef) + markInstruction(*MI, Flag, Worklist); + } else { + // For physical registers simply mark the defining instruction + markInstruction(*MI, Flag, Worklist); } } - } while (!ToProcess.empty()); -} -/// Mark all instructions defining the uses in \p MI with \p Flag. -void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag, - std::vector<WorkItem> &Worklist) { + if (!NextValue && !PhiStack.empty()) { + // Reach end of chain; revert to processing last phi + PhiEntry &Entry = PhiStack.back(); + NextValue = Entry.Phi; + NextPredIdx = Entry.PredIdx; + DefinedLanes = Entry.DefinedLanes; + PhiStack.pop_back(); + } - LLVM_DEBUG(dbgs() << "markInstructionUses " << PrintState(Flag) << ": " - << MI); + Value = NextValue; + } while (Value); +} - for (const MachineOperand &Use : MI.uses()) { - if (!Use.isReg() || !Use.isUse()) - continue; +void SIWholeQuadMode::markOperand(const MachineInstr &MI, + const MachineOperand &Op, char Flag, + std::vector<WorkItem> &Worklist) { + assert(Op.isReg()); + Register Reg = Op.getReg(); - Register Reg = Use.getReg(); + // Ignore some hardware registers + switch (Reg) { + case AMDGPU::EXEC: + case AMDGPU::EXEC_LO: + return; + default: + break; + } + LLVM_DEBUG(dbgs() << "markOperand " << PrintState(Flag) << ": " << Op + << " for " << MI); + if (Reg.isVirtual()) { + LiveRange &LR = LIS->getInterval(Reg); + markDefs(MI, LR, Reg, Op.getSubReg(), Flag, Worklist); + } else { // Handle physical registers that we need to track; this is mostly relevant // for VCC, which can appear as the (implicit) input of a uniform branch, // e.g. when a loop counter is stored in a VGPR. - if (!Reg.isVirtual()) { - if (Reg == AMDGPU::EXEC || Reg == AMDGPU::EXEC_LO) + for (MCRegUnitIterator RegUnit(Reg.asMCReg(), TRI); RegUnit.isValid(); + ++RegUnit) { + LiveRange &LR = LIS->getRegUnit(*RegUnit); + const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn(); + if (!Value) continue; - for (MCRegUnitIterator RegUnit(Reg.asMCReg(), TRI); RegUnit.isValid(); - ++RegUnit) { - LiveRange &LR = LIS->getRegUnit(*RegUnit); - const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn(); - if (!Value) - continue; + markDefs(MI, LR, *RegUnit, AMDGPU::NoSubRegister, Flag, Worklist); + } + } +} - if (MRI->isSSA()) { - // Since we're in machine SSA, we do not need to track physical - // registers across basic blocks. - if (Value->isPHIDef()) - continue; - markInstruction(*LIS->getInstructionFromIndex(Value->def), Flag, - Worklist); - } else { - markDefs(MI, LR, *RegUnit, AMDGPU::NoSubRegister, Flag, Worklist); - } - } +/// Mark all instructions defining the uses in \p MI with \p Flag. +void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag, + std::vector<WorkItem> &Worklist) { + LLVM_DEBUG(dbgs() << "markInstructionUses " << PrintState(Flag) << ": " + << MI); + for (const MachineOperand &Use : MI.uses()) { + if (!Use.isReg() || !Use.isUse()) continue; - } - - if (MRI->isSSA()) { - for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg())) - markInstruction(DefMI, Flag, Worklist); - } else { - LiveRange &LR = LIS->getInterval(Reg); - markDefs(MI, LR, Reg, Use.getSubReg(), Flag, Worklist); - } + markOperand(MI, Use, Flag, Worklist); } } @@ -392,6 +504,9 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, char Flags = 0; if (TII->isWQM(Opcode)) { + // If LOD is not supported WQM is not needed. + if (!ST->hasExtendedImageInsts()) + continue; // Sampling instructions don't need to produce results for all pixels // in a quad, they just require all inputs of a quad to have been // computed for derivatives. @@ -407,27 +522,31 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, LowerToCopyInstrs.push_back(&MI); SoftWQMInstrs.push_back(&MI); continue; - } else if (Opcode == AMDGPU::WWM) { - // The WWM intrinsic doesn't make the same guarantee, and plus it needs - // to be executed in WQM or Exact so that its copy doesn't clobber - // inactive lanes. - markInstructionUses(MI, StateWWM, Worklist); - GlobalFlags |= StateWWM; + } else if (Opcode == AMDGPU::STRICT_WWM) { + // The STRICT_WWM intrinsic doesn't make the same guarantee, and plus + // it needs to be executed in WQM or Exact so that its copy doesn't + // clobber inactive lanes. + markInstructionUses(MI, StateStrictWWM, Worklist); + GlobalFlags |= StateStrictWWM; + LowerToMovInstrs.push_back(&MI); + continue; + } else if (Opcode == AMDGPU::STRICT_WQM) { + // STRICT_WQM is similar to STRICTWWM, but instead of enabling all + // threads of the wave like STRICTWWM, STRICT_WQM enables all threads in + // quads that have at least one active thread. + markInstructionUses(MI, StateStrictWQM, Worklist); + GlobalFlags |= StateStrictWQM; LowerToMovInstrs.push_back(&MI); continue; } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 || Opcode == AMDGPU::V_SET_INACTIVE_B64) { - III.Disabled = StateWWM; + III.Disabled = StateStrict; MachineOperand &Inactive = MI.getOperand(2); if (Inactive.isReg()) { if (Inactive.isUndef()) { LowerToCopyInstrs.push_back(&MI); } else { - Register Reg = Inactive.getReg(); - if (Reg.isVirtual()) { - for (MachineInstr &DefMI : MRI->def_instructions(Reg)) - markInstruction(DefMI, StateWWM, Worklist); - } + markOperand(MI, Inactive, StateStrictWWM, Worklist); } } SetInactiveInstrs.push_back(&MI); @@ -439,15 +558,21 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, Worklist.push_back(&MBB); } GlobalFlags |= StateExact; - III.Disabled = StateWQM | StateWWM; + III.Disabled = StateWQM | StateStrict; continue; } else { - if (Opcode == AMDGPU::SI_PS_LIVE) { + if (Opcode == AMDGPU::SI_PS_LIVE || Opcode == AMDGPU::SI_LIVE_MASK) { LiveMaskQueries.push_back(&MI); + } else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR || + Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR || + Opcode == AMDGPU::SI_DEMOTE_I1) { + KillInstrs.push_back(&MI); + BBI.NeedsLowering = true; } else if (WQMOutputs) { // The function is in machine SSA form, which means that physical // VGPRs correspond to shader inputs and outputs. Inputs are // only used, outputs are only defined. + // FIXME: is this still valid? for (const MachineOperand &MO : MI.defs()) { if (!MO.isReg()) continue; @@ -510,7 +635,7 @@ void SIWholeQuadMode::propagateInstruction(MachineInstr &MI, // Propagate backwards within block if (MachineInstr *PrevMI = MI.getPrevNode()) { - char InNeeds = (II.Needs & ~StateWWM) | II.OutNeeds; + char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds; if (!PrevMI->isPHI()) { InstrInfo &PrevII = Instructions[PrevMI]; if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) { @@ -526,10 +651,12 @@ void SIWholeQuadMode::propagateInstruction(MachineInstr &MI, if (II.Needs != 0) markInstructionUses(MI, II.Needs, Worklist); - // Ensure we process a block containing WWM, even if it does not require any - // WQM transitions. - if (II.Needs & StateWWM) - BI.Needs |= StateWWM; + // Ensure we process a block containing StrictWWM/StrictWQM, even if it does + // not require any WQM transitions. + if (II.Needs & StateStrictWWM) + BI.Needs |= StateStrictWWM; + if (II.Needs & StateStrictWQM) + BI.Needs |= StateStrictWQM; } void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB, @@ -604,6 +731,339 @@ SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB, return Restore; } +MachineBasicBlock *SIWholeQuadMode::splitBlock(MachineBasicBlock *BB, + MachineInstr *TermMI) { + LLVM_DEBUG(dbgs() << "Split block " << printMBBReference(*BB) << " @ " + << *TermMI << "\n"); + + MachineBasicBlock *SplitBB = + BB->splitAt(*TermMI, /*UpdateLiveIns*/ true, LIS); + + // Convert last instruction in block to a terminator. + // Note: this only covers the expected patterns + unsigned NewOpcode = 0; + switch (TermMI->getOpcode()) { + case AMDGPU::S_AND_B32: + NewOpcode = AMDGPU::S_AND_B32_term; + break; + case AMDGPU::S_AND_B64: + NewOpcode = AMDGPU::S_AND_B64_term; + break; + case AMDGPU::S_MOV_B32: + NewOpcode = AMDGPU::S_MOV_B32_term; + break; + case AMDGPU::S_MOV_B64: + NewOpcode = AMDGPU::S_MOV_B64_term; + break; + default: + break; + } + if (NewOpcode) + TermMI->setDesc(TII->get(NewOpcode)); + + if (SplitBB != BB) { + // Update dominator trees + using DomTreeT = DomTreeBase<MachineBasicBlock>; + SmallVector<DomTreeT::UpdateType, 16> DTUpdates; + for (MachineBasicBlock *Succ : SplitBB->successors()) { + DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ}); + DTUpdates.push_back({DomTreeT::Delete, BB, Succ}); + } + DTUpdates.push_back({DomTreeT::Insert, BB, SplitBB}); + if (MDT) + MDT->getBase().applyUpdates(DTUpdates); + if (PDT) + PDT->getBase().applyUpdates(DTUpdates); + + // Link blocks + MachineInstr *MI = + BuildMI(*BB, BB->end(), DebugLoc(), TII->get(AMDGPU::S_BRANCH)) + .addMBB(SplitBB); + LIS->InsertMachineInstrInMaps(*MI); + } + + return SplitBB; +} + +MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB, + MachineInstr &MI) { + const DebugLoc &DL = MI.getDebugLoc(); + unsigned Opcode = 0; + + assert(MI.getOperand(0).isReg()); + + // Comparison is for live lanes; however here we compute the inverse + // (killed lanes). This is because VCMP will always generate 0 bits + // for inactive lanes so a mask of live lanes would not be correct + // inside control flow. + // Invert the comparison by swapping the operands and adjusting + // the comparison codes. + + switch (MI.getOperand(2).getImm()) { + case ISD::SETUEQ: + Opcode = AMDGPU::V_CMP_LG_F32_e64; + break; + case ISD::SETUGT: + Opcode = AMDGPU::V_CMP_GE_F32_e64; + break; + case ISD::SETUGE: + Opcode = AMDGPU::V_CMP_GT_F32_e64; + break; + case ISD::SETULT: + Opcode = AMDGPU::V_CMP_LE_F32_e64; + break; + case ISD::SETULE: + Opcode = AMDGPU::V_CMP_LT_F32_e64; + break; + case ISD::SETUNE: + Opcode = AMDGPU::V_CMP_EQ_F32_e64; + break; + case ISD::SETO: + Opcode = AMDGPU::V_CMP_O_F32_e64; + break; + case ISD::SETUO: + Opcode = AMDGPU::V_CMP_U_F32_e64; + break; + case ISD::SETOEQ: + case ISD::SETEQ: + Opcode = AMDGPU::V_CMP_NEQ_F32_e64; + break; + case ISD::SETOGT: + case ISD::SETGT: + Opcode = AMDGPU::V_CMP_NLT_F32_e64; + break; + case ISD::SETOGE: + case ISD::SETGE: + Opcode = AMDGPU::V_CMP_NLE_F32_e64; + break; + case ISD::SETOLT: + case ISD::SETLT: + Opcode = AMDGPU::V_CMP_NGT_F32_e64; + break; + case ISD::SETOLE: + case ISD::SETLE: + Opcode = AMDGPU::V_CMP_NGE_F32_e64; + break; + case ISD::SETONE: + case ISD::SETNE: + Opcode = AMDGPU::V_CMP_NLG_F32_e64; + break; + default: + llvm_unreachable("invalid ISD:SET cond code"); + } + + // Pick opcode based on comparison type. + MachineInstr *VcmpMI; + const MachineOperand &Op0 = MI.getOperand(0); + const MachineOperand &Op1 = MI.getOperand(1); + if (TRI->isVGPR(*MRI, Op0.getReg())) { + Opcode = AMDGPU::getVOPe32(Opcode); + VcmpMI = BuildMI(MBB, &MI, DL, TII->get(Opcode)).add(Op1).add(Op0); + } else { + VcmpMI = BuildMI(MBB, &MI, DL, TII->get(Opcode)) + .addReg(AMDGPU::VCC, RegState::Define) + .addImm(0) // src0 modifiers + .add(Op1) + .addImm(0) // src1 modifiers + .add(Op0) + .addImm(0); // omod + } + + // VCC represents lanes killed. + Register VCC = ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC; + + MachineInstr *MaskUpdateMI = + BuildMI(MBB, MI, DL, TII->get(AndN2Opc), LiveMaskReg) + .addReg(LiveMaskReg) + .addReg(VCC); + + // State of SCC represents whether any lanes are live in mask, + // if SCC is 0 then no lanes will be alive anymore. + MachineInstr *EarlyTermMI = + BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_EARLY_TERMINATE_SCC0)); + + MachineInstr *ExecMaskMI = + BuildMI(MBB, MI, DL, TII->get(AndN2Opc), Exec).addReg(Exec).addReg(VCC); + + assert(MBB.succ_size() == 1); + MachineInstr *NewTerm = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_BRANCH)) + .addMBB(*MBB.succ_begin()); + + // Update live intervals + LIS->ReplaceMachineInstrInMaps(MI, *VcmpMI); + MBB.remove(&MI); + + LIS->InsertMachineInstrInMaps(*MaskUpdateMI); + LIS->InsertMachineInstrInMaps(*ExecMaskMI); + LIS->InsertMachineInstrInMaps(*EarlyTermMI); + LIS->InsertMachineInstrInMaps(*NewTerm); + + return NewTerm; +} + +MachineInstr *SIWholeQuadMode::lowerKillI1(MachineBasicBlock &MBB, + MachineInstr &MI, bool IsWQM) { + const DebugLoc &DL = MI.getDebugLoc(); + MachineInstr *MaskUpdateMI = nullptr; + + const bool IsDemote = IsWQM && (MI.getOpcode() == AMDGPU::SI_DEMOTE_I1); + const MachineOperand &Op = MI.getOperand(0); + int64_t KillVal = MI.getOperand(1).getImm(); + MachineInstr *ComputeKilledMaskMI = nullptr; + Register CndReg = !Op.isImm() ? Op.getReg() : Register(); + Register TmpReg; + + // Is this a static or dynamic kill? + if (Op.isImm()) { + if (Op.getImm() == KillVal) { + // Static: all active lanes are killed + MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(AndN2Opc), LiveMaskReg) + .addReg(LiveMaskReg) + .addReg(Exec); + } else { + // Static: kill does nothing + MachineInstr *NewTerm = nullptr; + if (MI.getOpcode() == AMDGPU::SI_DEMOTE_I1) { + LIS->RemoveMachineInstrFromMaps(MI); + } else { + assert(MBB.succ_size() == 1); + NewTerm = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_BRANCH)) + .addMBB(*MBB.succ_begin()); + LIS->ReplaceMachineInstrInMaps(MI, *NewTerm); + } + MBB.remove(&MI); + return NewTerm; + } + } else { + if (!KillVal) { + // Op represents live lanes after kill, + // so exec mask needs to be factored in. + TmpReg = MRI->createVirtualRegister(TRI->getBoolRC()); + ComputeKilledMaskMI = + BuildMI(MBB, MI, DL, TII->get(XorOpc), TmpReg).add(Op).addReg(Exec); + MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(AndN2Opc), LiveMaskReg) + .addReg(LiveMaskReg) + .addReg(TmpReg); + } else { + // Op represents lanes to kill + MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(AndN2Opc), LiveMaskReg) + .addReg(LiveMaskReg) + .add(Op); + } + } + + // State of SCC represents whether any lanes are live in mask, + // if SCC is 0 then no lanes will be alive anymore. + MachineInstr *EarlyTermMI = + BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_EARLY_TERMINATE_SCC0)); + + // In the case we got this far some lanes are still live, + // update EXEC to deactivate lanes as appropriate. + MachineInstr *NewTerm; + MachineInstr *WQMMaskMI = nullptr; + Register LiveMaskWQM; + if (IsDemote) { + // Demotes deactive quads with only helper lanes + LiveMaskWQM = MRI->createVirtualRegister(TRI->getBoolRC()); + WQMMaskMI = + BuildMI(MBB, MI, DL, TII->get(WQMOpc), LiveMaskWQM).addReg(LiveMaskReg); + NewTerm = BuildMI(MBB, MI, DL, TII->get(AndOpc), Exec) + .addReg(Exec) + .addReg(LiveMaskWQM); + } else { + // Kills deactivate lanes + if (Op.isImm()) { + unsigned MovOpc = ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; + NewTerm = BuildMI(MBB, &MI, DL, TII->get(MovOpc), Exec).addImm(0); + } else if (!IsWQM) { + NewTerm = BuildMI(MBB, &MI, DL, TII->get(AndOpc), Exec) + .addReg(Exec) + .addReg(LiveMaskReg); + } else { + unsigned Opcode = KillVal ? AndN2Opc : AndOpc; + NewTerm = + BuildMI(MBB, &MI, DL, TII->get(Opcode), Exec).addReg(Exec).add(Op); + } + } + + // Update live intervals + LIS->RemoveMachineInstrFromMaps(MI); + MBB.remove(&MI); + assert(EarlyTermMI); + assert(MaskUpdateMI); + assert(NewTerm); + if (ComputeKilledMaskMI) + LIS->InsertMachineInstrInMaps(*ComputeKilledMaskMI); + LIS->InsertMachineInstrInMaps(*MaskUpdateMI); + LIS->InsertMachineInstrInMaps(*EarlyTermMI); + if (WQMMaskMI) + LIS->InsertMachineInstrInMaps(*WQMMaskMI); + LIS->InsertMachineInstrInMaps(*NewTerm); + + if (CndReg) { + LIS->removeInterval(CndReg); + LIS->createAndComputeVirtRegInterval(CndReg); + } + if (TmpReg) + LIS->createAndComputeVirtRegInterval(TmpReg); + if (LiveMaskWQM) + LIS->createAndComputeVirtRegInterval(LiveMaskWQM); + + return NewTerm; +} + +// Replace (or supplement) instructions accessing live mask. +// This can only happen once all the live mask registers have been created +// and the execute state (WQM/StrictWWM/Exact) of instructions is known. +void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB) { + auto BII = Blocks.find(&MBB); + if (BII == Blocks.end()) + return; + + const BlockInfo &BI = BII->second; + if (!BI.NeedsLowering) + return; + + LLVM_DEBUG(dbgs() << "\nLowering block " << printMBBReference(MBB) << ":\n"); + + SmallVector<MachineInstr *, 4> SplitPoints; + char State = BI.InitialState; + + auto II = MBB.getFirstNonPHI(), IE = MBB.end(); + while (II != IE) { + auto Next = std::next(II); + MachineInstr &MI = *II; + + if (StateTransition.count(&MI)) + State = StateTransition[&MI]; + + MachineInstr *SplitPoint = nullptr; + switch (MI.getOpcode()) { + case AMDGPU::SI_DEMOTE_I1: + case AMDGPU::SI_KILL_I1_TERMINATOR: + SplitPoint = lowerKillI1(MBB, MI, State == StateWQM); + break; + case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: + SplitPoint = lowerKillF32(MBB, MI); + break; + default: + break; + } + if (SplitPoint) + SplitPoints.push_back(SplitPoint); + + II = Next; + } + + // Perform splitting after instruction scan to simplify iteration. + if (!SplitPoints.empty()) { + MachineBasicBlock *BB = &MBB; + for (MachineInstr *MI : SplitPoints) { + BB = splitBlock(BB, MI); + } + } +} + // Return an iterator in the (inclusive) range [First, Last] at which // instructions can be safely inserted, keeping in mind that some of the // instructions we want to add necessarily clobber SCC. @@ -680,93 +1140,108 @@ MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion( void SIWholeQuadMode::toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SaveWQM, unsigned LiveMaskReg) { + Register SaveWQM) { MachineInstr *MI; if (SaveWQM) { - MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ? - AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64), - SaveWQM) + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AndSaveExecOpc), SaveWQM) .addReg(LiveMaskReg); } else { - unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ? - AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64), - Exec) + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AndOpc), Exec) .addReg(Exec) .addReg(LiveMaskReg); } LIS->InsertMachineInstrInMaps(*MI); + StateTransition[MI] = StateExact; } void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, - unsigned SavedWQM) { + Register SavedWQM) { MachineInstr *MI; - unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; if (SavedWQM) { MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), Exec) .addReg(SavedWQM); } else { - MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ? - AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64), - Exec) - .addReg(Exec); + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(WQMOpc), Exec).addReg(Exec); } LIS->InsertMachineInstrInMaps(*MI); + StateTransition[MI] = StateWQM; } -void SIWholeQuadMode::toWWM(MachineBasicBlock &MBB, - MachineBasicBlock::iterator Before, - unsigned SaveOrig) { +void SIWholeQuadMode::toStrictMode(MachineBasicBlock &MBB, + MachineBasicBlock::iterator Before, + Register SaveOrig, char StrictStateNeeded) { MachineInstr *MI; - assert(SaveOrig); - MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_WWM), SaveOrig) - .addImm(-1); + assert(StrictStateNeeded == StateStrictWWM || + StrictStateNeeded == StateStrictWQM); + + if (StrictStateNeeded == StateStrictWWM) { + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WWM), + SaveOrig) + .addImm(-1); + } else { + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WQM), + SaveOrig) + .addImm(-1); + } LIS->InsertMachineInstrInMaps(*MI); + StateTransition[MI] = StateStrictWWM; } -void SIWholeQuadMode::fromWWM(MachineBasicBlock &MBB, - MachineBasicBlock::iterator Before, - unsigned SavedOrig) { +void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB, + MachineBasicBlock::iterator Before, + Register SavedOrig, char NonStrictState, + char CurrentStrictState) { MachineInstr *MI; assert(SavedOrig); - MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_WWM), - ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC) - .addReg(SavedOrig); + assert(CurrentStrictState == StateStrictWWM || + CurrentStrictState == StateStrictWQM); + + if (CurrentStrictState == StateStrictWWM) { + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WWM), + Exec) + .addReg(SavedOrig); + } else { + MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WQM), + Exec) + .addReg(SavedOrig); + } LIS->InsertMachineInstrInMaps(*MI); + StateTransition[MI] = NonStrictState; } -void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, - bool isEntry) { +void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) { auto BII = Blocks.find(&MBB); if (BII == Blocks.end()) return; - const BlockInfo &BI = BII->second; + BlockInfo &BI = BII->second; // This is a non-entry block that is WQM throughout, so no need to do // anything. - if (!isEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) + if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) { + BI.InitialState = StateWQM; return; + } LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB) << ":\n"); - unsigned SavedWQMReg = 0; - unsigned SavedNonWWMReg = 0; - bool WQMFromExec = isEntry; - char State = (isEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM; - char NonWWMState = 0; + Register SavedWQMReg; + Register SavedNonStrictReg; + bool WQMFromExec = IsEntry; + char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM; + char NonStrictState = 0; const TargetRegisterClass *BoolRC = TRI->getBoolRC(); auto II = MBB.getFirstNonPHI(), IE = MBB.end(); - if (isEntry) { + if (IsEntry) { // Skip the instruction that saves LiveMask if (II != IE && II->getOpcode() == AMDGPU::COPY) ++II; @@ -776,22 +1251,25 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, // Exact or vice versa. MachineBasicBlock::iterator FirstWQM = IE; - // This stores the first instruction where it's safe to switch from WWM to - // Exact/WQM or to switch to WWM. It must always be the same as, or after, - // FirstWQM since if it's safe to switch to/from WWM, it must be safe to - // switch to/from WQM as well. - MachineBasicBlock::iterator FirstWWM = IE; + // This stores the first instruction where it's safe to switch from Strict + // mode to Exact/WQM or to switch to Strict mode. It must always be the same + // as, or after, FirstWQM since if it's safe to switch to/from Strict, it must + // be safe to switch to/from WQM as well. + MachineBasicBlock::iterator FirstStrict = IE; + + // Record initial state is block information. + BI.InitialState = State; for (;;) { MachineBasicBlock::iterator Next = II; - char Needs = StateExact | StateWQM; // WWM is disabled by default + char Needs = StateExact | StateWQM; // Strict mode is disabled by default. char OutNeeds = 0; if (FirstWQM == IE) FirstWQM = II; - if (FirstWWM == IE) - FirstWWM = II; + if (FirstStrict == IE) + FirstStrict = II; // First, figure out the allowed states (Needs) based on the propagated // flags. @@ -801,8 +1279,10 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, if (MI.isTerminator() || TII->mayReadEXEC(*MRI, MI)) { auto III = Instructions.find(&MI); if (III != Instructions.end()) { - if (III->second.Needs & StateWWM) - Needs = StateWWM; + if (III->second.Needs & StateStrictWWM) + Needs = StateStrictWWM; + else if (III->second.Needs & StateStrictWQM) + Needs = StateStrictWQM; else if (III->second.Needs & StateWQM) Needs = StateWQM; else @@ -811,8 +1291,8 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, } } else { // If the instruction doesn't actually need a correct EXEC, then we can - // safely leave WWM enabled. - Needs = StateExact | StateWQM | StateWWM; + // safely leave Strict mode enabled. + Needs = StateExact | StateWQM | StateStrict; } if (MI.isTerminator() && OutNeeds == StateExact) @@ -832,32 +1312,56 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, // Now, transition if necessary. if (!(Needs & State)) { MachineBasicBlock::iterator First; - if (State == StateWWM || Needs == StateWWM) { - // We must switch to or from WWM - First = FirstWWM; + if (State == StateStrictWWM || Needs == StateStrictWWM || + State == StateStrictWQM || Needs == StateStrictWQM) { + // We must switch to or from Strict mode. + First = FirstStrict; } else { - // We only need to switch to/from WQM, so we can use FirstWQM + // We only need to switch to/from WQM, so we can use FirstWQM. First = FirstWQM; } + // Whether we need to save SCC depends on start and end states. + bool SaveSCC = false; + switch (State) { + case StateExact: + case StateStrictWWM: + case StateStrictWQM: + // Exact/Strict -> Strict: save SCC + // Exact/Strict -> WQM: save SCC if WQM mask is generated from exec + // Exact/Strict -> Exact: no save + SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec); + break; + case StateWQM: + // WQM -> Exact/Strict: save SCC + SaveSCC = !(Needs & StateWQM); + break; + default: + llvm_unreachable("Unknown state"); + break; + } MachineBasicBlock::iterator Before = - prepareInsertion(MBB, First, II, Needs == StateWQM, - Needs == StateExact || WQMFromExec); + prepareInsertion(MBB, First, II, Needs == StateWQM, SaveSCC); + + if (State & StateStrict) { + assert(State == StateStrictWWM || State == StateStrictWQM); + assert(SavedNonStrictReg); + fromStrictMode(MBB, Before, SavedNonStrictReg, NonStrictState, State); - if (State == StateWWM) { - assert(SavedNonWWMReg); - fromWWM(MBB, Before, SavedNonWWMReg); - LIS->createAndComputeVirtRegInterval(SavedNonWWMReg); - SavedNonWWMReg = 0; - State = NonWWMState; + LIS->createAndComputeVirtRegInterval(SavedNonStrictReg); + SavedNonStrictReg = 0; + State = NonStrictState; } - if (Needs == StateWWM) { - NonWWMState = State; - assert(!SavedNonWWMReg); - SavedNonWWMReg = MRI->createVirtualRegister(BoolRC); - toWWM(MBB, Before, SavedNonWWMReg); - State = StateWWM; + if (Needs & StateStrict) { + NonStrictState = State; + assert(Needs == StateStrictWWM || Needs == StateStrictWQM); + assert(!SavedNonStrictReg); + SavedNonStrictReg = MRI->createVirtualRegister(BoolRC); + + toStrictMode(MBB, Before, SavedNonStrictReg, Needs); + State = Needs; + } else { if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) { if (!WQMFromExec && (OutNeeds & StateWQM)) { @@ -865,7 +1369,7 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, SavedWQMReg = MRI->createVirtualRegister(BoolRC); } - toExact(MBB, Before, SavedWQMReg, LiveMaskReg); + toExact(MBB, Before, SavedWQMReg); State = StateExact; } else if (State == StateExact && (Needs & StateWQM) && !(Needs & StateExact)) { @@ -879,17 +1383,18 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, } State = StateWQM; } else { - // We can get here if we transitioned from WWM to a non-WWM state that - // already matches our needs, but we shouldn't need to do anything. + // We can get here if we transitioned from StrictWWM to a + // non-StrictWWM state that already matches our needs, but we + // shouldn't need to do anything. assert(Needs & State); } } } - if (Needs != (StateExact | StateWQM | StateWWM)) { + if (Needs != (StateExact | StateWQM | StateStrict)) { if (Needs != (StateExact | StateWQM)) FirstWQM = IE; - FirstWWM = IE; + FirstStrict = IE; } if (II == IE) @@ -898,10 +1403,10 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, II = Next; } assert(!SavedWQMReg); - assert(!SavedNonWWMReg); + assert(!SavedNonStrictReg); } -void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) { +void SIWholeQuadMode::lowerLiveMaskQueries() { for (MachineInstr *MI : LiveMaskQueries) { const DebugLoc &DL = MI->getDebugLoc(); Register Dest = MI->getOperand(0).getReg(); @@ -931,9 +1436,12 @@ void SIWholeQuadMode::lowerCopyInstrs() { const unsigned MovOp = TII->getMovOpcode(regClass); MI->setDesc(TII->get(MovOp)); - // And make it implicitly depend on exec (like all VALU movs should do). - MI->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); - } else if (!MRI->isSSA()) { + // Check that it already implicitly depends on exec (like all VALU movs + // should do). + assert(any_of(MI->implicit_operands(), [](const MachineOperand &MO) { + return MO.isUse() && MO.getReg() == AMDGPU::EXEC; + })); + } else { // Remove early-clobber and exec dependency from simple SGPR copies. // This allows some to be eliminated during/post RA. LLVM_DEBUG(dbgs() << "simplify SGPR copy: " << *MI); @@ -969,13 +1477,38 @@ void SIWholeQuadMode::lowerCopyInstrs() { } } +void SIWholeQuadMode::lowerKillInstrs(bool IsWQM) { + for (MachineInstr *MI : KillInstrs) { + MachineBasicBlock *MBB = MI->getParent(); + MachineInstr *SplitPoint = nullptr; + switch (MI->getOpcode()) { + case AMDGPU::SI_DEMOTE_I1: + case AMDGPU::SI_KILL_I1_TERMINATOR: + SplitPoint = lowerKillI1(*MBB, *MI, IsWQM); + break; + case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: + SplitPoint = lowerKillF32(*MBB, *MI); + break; + default: + continue; + } + if (SplitPoint) + splitBlock(MBB, SplitPoint); + } +} + bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { + LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName() + << " ------------- \n"); + LLVM_DEBUG(MF.dump();); + Instructions.clear(); Blocks.clear(); LiveMaskQueries.clear(); LowerToCopyInstrs.clear(); LowerToMovInstrs.clear(); - CallingConv = MF.getFunction().getCallingConv(); + KillInstrs.clear(); + StateTransition.clear(); ST = &MF.getSubtarget<GCNSubtarget>(); @@ -983,64 +1516,72 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { TRI = &TII->getRegisterInfo(); MRI = &MF.getRegInfo(); LIS = &getAnalysis<LiveIntervals>(); + MDT = &getAnalysis<MachineDominatorTree>(); + PDT = &getAnalysis<MachinePostDominatorTree>(); if (ST->isWave32()) { AndOpc = AMDGPU::S_AND_B32; - XorTermrOpc = AMDGPU::S_XOR_B32_term; + AndN2Opc = AMDGPU::S_ANDN2_B32; + XorOpc = AMDGPU::S_XOR_B32; + AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B32; OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B32; + WQMOpc = AMDGPU::S_WQM_B32; Exec = AMDGPU::EXEC_LO; } else { AndOpc = AMDGPU::S_AND_B64; - XorTermrOpc = AMDGPU::S_XOR_B64_term; + AndN2Opc = AMDGPU::S_ANDN2_B64; + XorOpc = AMDGPU::S_XOR_B64; + AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B64; OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B64; + WQMOpc = AMDGPU::S_WQM_B64; Exec = AMDGPU::EXEC; } - char GlobalFlags = analyzeFunction(MF); - unsigned LiveMaskReg = 0; - if (!(GlobalFlags & StateWQM)) { - lowerLiveMaskQueries(Exec); - if (!(GlobalFlags & StateWWM) && LowerToCopyInstrs.empty() && LowerToMovInstrs.empty()) - return !LiveMaskQueries.empty(); - } else { - // Store a copy of the original live mask when required - MachineBasicBlock &Entry = MF.front(); - MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI(); + const char GlobalFlags = analyzeFunction(MF); + const bool NeedsLiveMask = !(KillInstrs.empty() && LiveMaskQueries.empty()); - if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) { - LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC()); - MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(), - TII->get(AMDGPU::COPY), LiveMaskReg) - .addReg(Exec); - LIS->InsertMachineInstrInMaps(*MI); - } + LiveMaskReg = Exec; - lowerLiveMaskQueries(LiveMaskReg); + // Shader is simple does not need any state changes or any complex lowering + if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() && + LowerToMovInstrs.empty() && KillInstrs.empty()) { + lowerLiveMaskQueries(); + return !LiveMaskQueries.empty(); + } - if (GlobalFlags == StateWQM) { - // For a shader that needs only WQM, we can just set it once. - auto MI = BuildMI(Entry, EntryMI, DebugLoc(), - TII->get(ST->isWave32() ? AMDGPU::S_WQM_B32 - : AMDGPU::S_WQM_B64), - Exec) - .addReg(Exec); - LIS->InsertMachineInstrInMaps(*MI); + MachineBasicBlock &Entry = MF.front(); + MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI(); - lowerCopyInstrs(); - // EntryMI may become invalid here - return true; - } + // Store a copy of the original live mask when required + if (NeedsLiveMask || (GlobalFlags & StateWQM)) { + LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC()); + MachineInstr *MI = + BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::COPY), LiveMaskReg) + .addReg(Exec); + LIS->InsertMachineInstrInMaps(*MI); } LLVM_DEBUG(printInfo()); + lowerLiveMaskQueries(); lowerCopyInstrs(); - // Handle the general case - for (auto BII : Blocks) - processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin()); + // Shader only needs WQM + if (GlobalFlags == StateWQM) { + auto MI = BuildMI(Entry, EntryMI, DebugLoc(), TII->get(WQMOpc), Exec) + .addReg(Exec); + LIS->InsertMachineInstrInMaps(*MI); + lowerKillInstrs(true); + } else { + for (auto BII : Blocks) + processBlock(*BII.first, BII.first == &Entry); + // Lowering blocks causes block splitting so perform as a second pass. + for (auto BII : Blocks) + lowerBlock(*BII.first); + } - if (LiveMaskReg) + // Compute live range for live mask + if (LiveMaskReg != Exec) LIS->createAndComputeVirtRegInterval(LiveMaskReg); // Physical registers like SCC aren't tracked by default anyway, so just @@ -1048,5 +1589,9 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { // the analysis results. LIS->removeRegUnit(*MCRegUnitIterator(MCRegister::from(AMDGPU::SCC), TRI)); + // If we performed any kills then recompute EXEC + if (!KillInstrs.empty()) + LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::EXEC, TRI)); + return true; } |