diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 2116 |
1 files changed, 1593 insertions, 523 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 1bb01dc8fa11..dfaf97bfb08e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -8,10 +8,69 @@ /// \file /// This file implements the targeting of the RegisterBankInfo class for /// AMDGPU. -/// \todo This should be generated by TableGen. +/// +/// \par +/// +/// AMDGPU has unique register bank constraints that require special high level +/// strategies to deal with. There are two main true physical register banks +/// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a +/// sort of pseudo-register bank needed to represent SGPRs used in a vector +/// boolean context. There is also the AGPR bank, which is a special purpose +/// physical register bank present on some subtargets. +/// +/// Copying from VGPR to SGPR is generally illegal, unless the value is known to +/// be uniform. It is generally not valid to legalize operands by inserting +/// copies as on other targets. Operations which require uniform, SGPR operands +/// generally require scalarization by repeatedly executing the instruction, +/// activating each set of lanes using a unique set of input values. This is +/// referred to as a waterfall loop. +/// +/// \par Booleans +/// +/// Booleans (s1 values) requires special consideration. A vector compare result +/// is naturally a bitmask with one bit per lane, in a 32 or 64-bit +/// register. These are represented with the VCC bank. During selection, we need +/// to be able to unambiguously go back from a register class to a register +/// bank. To distinguish whether an SGPR should use the SGPR or VCC register +/// bank, we need to know the use context type. An SGPR s1 value always means a +/// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets +/// SCC, which is a 1-bit unaddressable register. This will need to be copied to +/// a 32-bit virtual register. Taken together, this means we need to adjust the +/// type of boolean operations to be regbank legal. All SALU booleans need to be +/// widened to 32-bits, and all VALU booleans need to be s1 values. +/// +/// A noteworthy exception to the s1-means-vcc rule is for legalization artifact +/// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc +/// bank. A non-boolean source (such as a truncate from a 1-bit load from +/// memory) will require a copy to the VCC bank which will require clearing the +/// high bits and inserting a compare. +/// +/// \par Constant bus restriction +/// +/// VALU instructions have a limitation known as the constant bus +/// restriction. Most VALU instructions can use SGPR operands, but may read at +/// most 1 SGPR or constant literal value (this to 2 in gfx10 for most +/// instructions). This is one unique SGPR, so the same SGPR may be used for +/// multiple operands. From a register bank perspective, any combination of +/// operands should be legal as an SGPR, but this is contextually dependent on +/// the SGPR operands all being the same register. There is therefore optimal to +/// choose the SGPR with the most uses to minimize the number of copies. +/// +/// We avoid trying to solve this problem in RegBankSelect. Any VALU G_* +/// operation should have its source operands all mapped to VGPRs (except for +/// VCC), inserting copies from any SGPR operands. This the most trival legal +/// mapping. Anything beyond the simplest 1:1 instruction selection would be too +/// complicated to solve here. Every optimization pattern or instruction +/// selected to multiple outputs would have to enforce this rule, and there +/// would be additional complexity in tracking this rule for every G_* +/// operation. By forcing all inputs to VGPRs, it also simplifies the task of +/// picking the optimal operand combination from a post-isel optimization pass. +/// //===----------------------------------------------------------------------===// #include "AMDGPURegisterBankInfo.h" + +#include "AMDGPUGlobalISelUtils.h" #include "AMDGPUInstrInfo.h" #include "AMDGPUSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" @@ -19,8 +78,8 @@ #include "SIRegisterInfo.h" #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -101,8 +160,9 @@ public: if (!Op.isReg()) continue; + // We may see physical registers if building a real MI Register Reg = Op.getReg(); - if (MRI.getRegClassOrRegBank(Reg)) + if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg)) continue; const RegisterBank *RB = NewBank; @@ -138,15 +198,16 @@ AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST) TII(Subtarget.getInstrInfo()) { // HACK: Until this is fully tablegen'd. - static bool AlreadyInit = false; - if (AlreadyInit) - return; + static llvm::once_flag InitializeRegisterBankFlag; - AlreadyInit = true; + static auto InitializeRegisterBankOnce = [this]() { + assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank && + &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank && + &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank); + (void)this; + }; - assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank && - &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank && - &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank); + llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce); } static bool isVectorRegisterBank(const RegisterBank &Bank) { @@ -159,7 +220,7 @@ unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, unsigned Size) const { // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane? if (Dst.getID() == AMDGPU::SGPRRegBankID && - isVectorRegisterBank(Src)) { + (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) { return std::numeric_limits<unsigned>::max(); } @@ -177,9 +238,6 @@ unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, Src.getID() == AMDGPU::VCCRegBankID)) return std::numeric_limits<unsigned>::max(); - if (Src.getID() == AMDGPU::VCCRegBankID) - return std::numeric_limits<unsigned>::max(); - // There is no direct copy between AGPRs. if (Dst.getID() == AMDGPU::AGPRRegBankID && Src.getID() == AMDGPU::AGPRRegBankID) @@ -317,22 +375,6 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects( const MachineInstr &MI, const MachineRegisterInfo &MRI) const { switch (MI.getIntrinsicID()) { - case Intrinsic::amdgcn_buffer_load: { - static const OpRegBankEntry<3> Table[4] = { - // Perfectly legal. - { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, - { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, - - // Waterfall loop needed for rsrc. In the worst case this will execute - // approximately an extra 10 * wavesize + 2 instructions. - { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1000 }, - { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1000 } - }; - - // rsrc, voffset, offset - const std::array<unsigned, 3> RegSrcOpIdx = { { 2, 3, 4 } }; - return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); - } case Intrinsic::amdgcn_s_buffer_load: { static const OpRegBankEntry<2> Table[4] = { // Perfectly legal. @@ -402,15 +444,15 @@ static bool isScalarLoadLegal(const MachineInstr &MI) { AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; // There are no extending SMRD/SMEM loads, and they require 4-byte alignment. - return MMO->getSize() >= 4 && MMO->getAlignment() >= 4 && - // Can't do a scalar atomic load. - !MMO->isAtomic() && - // Don't use scalar loads for volatile accesses to non-constant address - // spaces. - (IsConst || !MMO->isVolatile()) && - // Memory must be known constant, or not written before this load. - (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) && - AMDGPUInstrInfo::isUniformMMO(MMO); + return MMO->getSize() >= 4 && MMO->getAlign() >= Align(4) && + // Can't do a scalar atomic load. + !MMO->isAtomic() && + // Don't use scalar loads for volatile accesses to non-constant address + // spaces. + (IsConst || !MMO->isVolatile()) && + // Memory must be known constant, or not written before this load. + (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) && + AMDGPUInstrInfo::isUniformMMO(MMO); } RegisterBankInfo::InstructionMappings @@ -490,24 +532,6 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), 3); // Num Operands AltMappings.push_back(&VVMapping); - - const InstructionMapping &SVMapping = getInstructionMapping( - 3, 3, getOperandsMapping( - {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), - AMDGPU::getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), - 3); // Num Operands - AltMappings.push_back(&SVMapping); - - // SGPR in LHS is slightly preferrable, so make it VS more expensive than - // SV. - const InstructionMapping &VSMapping = getInstructionMapping( - 3, 4, getOperandsMapping( - {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), - AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), - AMDGPU::getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size)}), - 3); // Num Operands - AltMappings.push_back(&VSMapping); break; } case TargetOpcode::G_LOAD: @@ -517,7 +541,6 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( LLT PtrTy = MRI.getType(MI.getOperand(1).getReg()); unsigned PtrSize = PtrTy.getSizeInBits(); unsigned AS = PtrTy.getAddressSpace(); - LLT LoadTy = MRI.getType(MI.getOperand(0).getReg()); if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS && AS != AMDGPUAS::PRIVATE_ADDRESS) && @@ -531,9 +554,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( } const InstructionMapping &VVMapping = getInstructionMapping( - 2, 1, getOperandsMapping( - {AMDGPU::getValueMappingLoadSGPROnly(AMDGPU::VGPRRegBankID, LoadTy), - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}), + 2, 1, + getOperandsMapping( + {AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), + AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}), 2); // Num Operands AltMappings.push_back(&VVMapping); @@ -546,43 +570,6 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( return AltMappings; } - case TargetOpcode::G_ICMP: { - // TODO: Should report 32-bit for scalar output type. - unsigned Size = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI); - const InstructionMapping &SSMapping = getInstructionMapping(1, 1, - getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), - nullptr, // Predicate operand. - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), - 4); // Num Operands - AltMappings.push_back(&SSMapping); - - const InstructionMapping &SVMapping = getInstructionMapping(2, 1, - getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), - nullptr, // Predicate operand. - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}), - 4); // Num Operands - AltMappings.push_back(&SVMapping); - - const InstructionMapping &VSMapping = getInstructionMapping(3, 1, - getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), - nullptr, // Predicate operand. - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), - 4); // Num Operands - AltMappings.push_back(&VSMapping); - - const InstructionMapping &VVMapping = getInstructionMapping(4, 1, - getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), - nullptr, // Predicate operand. - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}), - 4); // Num Operands - AltMappings.push_back(&VVMapping); - - return AltMappings; - } case TargetOpcode::G_SELECT: { unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); const InstructionMapping &SSMapping = getInstructionMapping(1, 1, @@ -607,10 +594,8 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( case TargetOpcode::G_SMAX: case TargetOpcode::G_UMIN: case TargetOpcode::G_UMAX: { - static const OpRegBankEntry<3> Table[4] = { + static const OpRegBankEntry<3> Table[2] = { { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, - { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, - { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, // Scalar requires cmp+select, and extends if 16-bit. // FIXME: Should there be separate costs for 32 and 16-bit @@ -740,6 +725,10 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( SmallVector<Register, 4> InitResultRegs; SmallVector<Register, 4> PhiRegs; + // Track use registers which have already been expanded with a readfirstlane + // sequence. This may have multiple uses if moving a sequence. + DenseMap<Register, Register> WaterfalledRegMap; + MachineBasicBlock &MBB = B.getMBB(); MachineFunction *MF = &B.getMF(); @@ -755,6 +744,10 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( const unsigned ExecReg = Subtarget.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; +#ifndef NDEBUG + const int OrigRangeSize = std::distance(Range.begin(), Range.end()); +#endif + for (MachineInstr &MI : Range) { for (MachineOperand &Def : MI.defs()) { LLT ResTy = MRI.getType(Def.getReg()); @@ -820,13 +813,14 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( const DebugLoc &DL = B.getDL(); - // Figure out the iterator range after splicing the instructions. - auto NewBegin = std::prev(LoopBB->end()); + MachineInstr &FirstInst = *Range.begin(); // Move the instruction into the loop. Note we moved everything after // Range.end() already into a new block, so Range.end() is no longer valid. LoopBB->splice(LoopBB->end(), &MBB, Range.begin(), MBB.end()); + // Figure out the iterator range after splicing the instructions. + MachineBasicBlock::iterator NewBegin = FirstInst.getIterator(); auto NewEnd = LoopBB->end(); MachineBasicBlock::iterator I = Range.begin(); @@ -834,158 +828,173 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( Register CondReg; + assert(std::distance(NewBegin, NewEnd) == OrigRangeSize); + for (MachineInstr &MI : make_range(NewBegin, NewEnd)) { for (MachineOperand &Op : MI.uses()) { if (!Op.isReg() || Op.isDef()) continue; - if (SGPROperandRegs.count(Op.getReg())) { - LLT OpTy = MRI.getType(Op.getReg()); - unsigned OpSize = OpTy.getSizeInBits(); + Register OldReg = Op.getReg(); + if (!SGPROperandRegs.count(OldReg)) + continue; - // Can only do a readlane of 32-bit pieces. - if (OpSize == 32) { - // Avoid extra copies in the simple case of one 32-bit register. - Register CurrentLaneOpReg - = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); - MRI.setType(CurrentLaneOpReg, OpTy); + // See if we already processed this register in another instruction in the + // sequence. + auto OldVal = WaterfalledRegMap.find(OldReg); + if (OldVal != WaterfalledRegMap.end()) { + Op.setReg(OldVal->second); + continue; + } - constrainGenericRegister(Op.getReg(), AMDGPU::VGPR_32RegClass, MRI); - // Read the next variant <- also loop target. - BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), - CurrentLaneOpReg) - .addReg(Op.getReg()); + LLT OpTy = MRI.getType(Op.getReg()); + unsigned OpSize = OpTy.getSizeInBits(); - Register NewCondReg = MRI.createVirtualRegister(WaveRC); - bool First = CondReg == AMDGPU::NoRegister; - if (First) - CondReg = NewCondReg; + // Can only do a readlane of 32-bit pieces. + if (OpSize == 32) { + // Avoid extra copies in the simple case of one 32-bit register. + Register CurrentLaneOpReg + = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); + MRI.setType(CurrentLaneOpReg, OpTy); - // Compare the just read M0 value to all possible Idx values. - B.buildInstr(AMDGPU::V_CMP_EQ_U32_e64) - .addDef(NewCondReg) - .addReg(CurrentLaneOpReg) - .addReg(Op.getReg()); - Op.setReg(CurrentLaneOpReg); + constrainGenericRegister(Op.getReg(), AMDGPU::VGPR_32RegClass, MRI); + // Read the next variant <- also loop target. + BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), + CurrentLaneOpReg) + .addReg(Op.getReg()); - if (!First) { - Register AndReg = MRI.createVirtualRegister(WaveRC); + Register NewCondReg = MRI.createVirtualRegister(WaveRC); + bool First = CondReg == AMDGPU::NoRegister; + if (First) + CondReg = NewCondReg; - // If there are multiple operands to consider, and the conditions. - B.buildInstr(WaveAndOpc) - .addDef(AndReg) - .addReg(NewCondReg) - .addReg(CondReg); - CondReg = AndReg; - } - } else { - LLT S32 = LLT::scalar(32); - SmallVector<Register, 8> ReadlanePieces; + // Compare the just read M0 value to all possible Idx values. + B.buildInstr(AMDGPU::V_CMP_EQ_U32_e64) + .addDef(NewCondReg) + .addReg(CurrentLaneOpReg) + .addReg(Op.getReg()); + Op.setReg(CurrentLaneOpReg); - // The compares can be done as 64-bit, but the extract needs to be done - // in 32-bit pieces. + if (!First) { + Register AndReg = MRI.createVirtualRegister(WaveRC); + + // If there are multiple operands to consider, and the conditions. + B.buildInstr(WaveAndOpc) + .addDef(AndReg) + .addReg(NewCondReg) + .addReg(CondReg); + CondReg = AndReg; + } + } else { + LLT S32 = LLT::scalar(32); + SmallVector<Register, 8> ReadlanePieces; - bool Is64 = OpSize % 64 == 0; + // The compares can be done as 64-bit, but the extract needs to be done + // in 32-bit pieces. - LLT UnmergeTy = OpSize % 64 == 0 ? LLT::scalar(64) : LLT::scalar(32); - unsigned CmpOp = OpSize % 64 == 0 ? AMDGPU::V_CMP_EQ_U64_e64 - : AMDGPU::V_CMP_EQ_U32_e64; + bool Is64 = OpSize % 64 == 0; - // The compares can be done as 64-bit, but the extract needs to be done - // in 32-bit pieces. + LLT UnmergeTy = OpSize % 64 == 0 ? LLT::scalar(64) : LLT::scalar(32); + unsigned CmpOp = OpSize % 64 == 0 ? AMDGPU::V_CMP_EQ_U64_e64 + : AMDGPU::V_CMP_EQ_U32_e64; - // Insert the unmerge before the loop. + // The compares can be done as 64-bit, but the extract needs to be done + // in 32-bit pieces. - B.setMBB(MBB); - auto Unmerge = B.buildUnmerge(UnmergeTy, Op.getReg()); - B.setInstr(*I); + // Insert the unmerge before the loop. - unsigned NumPieces = Unmerge->getNumOperands() - 1; - for (unsigned PieceIdx = 0; PieceIdx != NumPieces; ++PieceIdx) { - Register UnmergePiece = Unmerge.getReg(PieceIdx); + B.setMBB(MBB); + auto Unmerge = B.buildUnmerge(UnmergeTy, Op.getReg()); + B.setInstr(*I); - Register CurrentLaneOpReg; - if (Is64) { - Register CurrentLaneOpRegLo = MRI.createGenericVirtualRegister(S32); - Register CurrentLaneOpRegHi = MRI.createGenericVirtualRegister(S32); + unsigned NumPieces = Unmerge->getNumOperands() - 1; + for (unsigned PieceIdx = 0; PieceIdx != NumPieces; ++PieceIdx) { + Register UnmergePiece = Unmerge.getReg(PieceIdx); - MRI.setRegClass(UnmergePiece, &AMDGPU::VReg_64RegClass); - MRI.setRegClass(CurrentLaneOpRegLo, &AMDGPU::SReg_32_XM0RegClass); - MRI.setRegClass(CurrentLaneOpRegHi, &AMDGPU::SReg_32_XM0RegClass); + Register CurrentLaneOpReg; + if (Is64) { + Register CurrentLaneOpRegLo = MRI.createGenericVirtualRegister(S32); + Register CurrentLaneOpRegHi = MRI.createGenericVirtualRegister(S32); - // Read the next variant <- also loop target. - BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), - CurrentLaneOpRegLo) - .addReg(UnmergePiece, 0, AMDGPU::sub0); + MRI.setRegClass(UnmergePiece, &AMDGPU::VReg_64RegClass); + MRI.setRegClass(CurrentLaneOpRegLo, &AMDGPU::SReg_32_XM0RegClass); + MRI.setRegClass(CurrentLaneOpRegHi, &AMDGPU::SReg_32_XM0RegClass); - // Read the next variant <- also loop target. - BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), - CurrentLaneOpRegHi) - .addReg(UnmergePiece, 0, AMDGPU::sub1); + // Read the next variant <- also loop target. + BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), + CurrentLaneOpRegLo) + .addReg(UnmergePiece, 0, AMDGPU::sub0); - CurrentLaneOpReg = - B.buildMerge(LLT::scalar(64), - {CurrentLaneOpRegLo, CurrentLaneOpRegHi}) - .getReg(0); + // Read the next variant <- also loop target. + BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), + CurrentLaneOpRegHi) + .addReg(UnmergePiece, 0, AMDGPU::sub1); - MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_64_XEXECRegClass); + CurrentLaneOpReg = + B.buildMerge(LLT::scalar(64), + {CurrentLaneOpRegLo, CurrentLaneOpRegHi}) + .getReg(0); - if (OpTy.getScalarSizeInBits() == 64) { - // If we need to produce a 64-bit element vector, so use the - // merged pieces - ReadlanePieces.push_back(CurrentLaneOpReg); - } else { - // 32-bit element type. - ReadlanePieces.push_back(CurrentLaneOpRegLo); - ReadlanePieces.push_back(CurrentLaneOpRegHi); - } - } else { - CurrentLaneOpReg = MRI.createGenericVirtualRegister(S32); - MRI.setRegClass(UnmergePiece, &AMDGPU::VGPR_32RegClass); - MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_32_XM0RegClass); + MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_64_XEXECRegClass); - // Read the next variant <- also loop target. - BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), - CurrentLaneOpReg) - .addReg(UnmergePiece); + if (OpTy.getScalarSizeInBits() == 64) { + // If we need to produce a 64-bit element vector, so use the + // merged pieces ReadlanePieces.push_back(CurrentLaneOpReg); + } else { + // 32-bit element type. + ReadlanePieces.push_back(CurrentLaneOpRegLo); + ReadlanePieces.push_back(CurrentLaneOpRegHi); } + } else { + CurrentLaneOpReg = MRI.createGenericVirtualRegister(S32); + MRI.setRegClass(UnmergePiece, &AMDGPU::VGPR_32RegClass); + MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_32_XM0RegClass); - Register NewCondReg = MRI.createVirtualRegister(WaveRC); - bool First = CondReg == AMDGPU::NoRegister; - if (First) - CondReg = NewCondReg; - - B.buildInstr(CmpOp) - .addDef(NewCondReg) - .addReg(CurrentLaneOpReg) + // Read the next variant <- also loop target. + BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), + CurrentLaneOpReg) .addReg(UnmergePiece); + ReadlanePieces.push_back(CurrentLaneOpReg); + } - if (!First) { - Register AndReg = MRI.createVirtualRegister(WaveRC); + Register NewCondReg = MRI.createVirtualRegister(WaveRC); + bool First = CondReg == AMDGPU::NoRegister; + if (First) + CondReg = NewCondReg; - // If there are multiple operands to consider, and the conditions. - B.buildInstr(WaveAndOpc) - .addDef(AndReg) - .addReg(NewCondReg) - .addReg(CondReg); - CondReg = AndReg; - } - } + B.buildInstr(CmpOp) + .addDef(NewCondReg) + .addReg(CurrentLaneOpReg) + .addReg(UnmergePiece); - // FIXME: Build merge seems to switch to CONCAT_VECTORS but not - // BUILD_VECTOR - if (OpTy.isVector()) { - auto Merge = B.buildBuildVector(OpTy, ReadlanePieces); - Op.setReg(Merge.getReg(0)); - } else { - auto Merge = B.buildMerge(OpTy, ReadlanePieces); - Op.setReg(Merge.getReg(0)); + if (!First) { + Register AndReg = MRI.createVirtualRegister(WaveRC); + + // If there are multiple operands to consider, and the conditions. + B.buildInstr(WaveAndOpc) + .addDef(AndReg) + .addReg(NewCondReg) + .addReg(CondReg); + CondReg = AndReg; } + } - MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank); + // FIXME: Build merge seems to switch to CONCAT_VECTORS but not + // BUILD_VECTOR + if (OpTy.isVector()) { + auto Merge = B.buildBuildVector(OpTy, ReadlanePieces); + Op.setReg(Merge.getReg(0)); + } else { + auto Merge = B.buildMerge(OpTy, ReadlanePieces); + Op.setReg(Merge.getReg(0)); } + + MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank); } + + // Make sure we don't re-process this register again. + WaterfalledRegMap.insert(std::make_pair(OldReg, Op.getReg())); } } @@ -1093,53 +1102,89 @@ void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane( MI.getOperand(OpIdx).setReg(SGPR); } -// When regbankselect repairs registers, it will insert a repair instruction -// which defines the repaired register. Then it calls applyMapping and expects -// that the targets will either delete or rewrite the originally wrote to the -// repaired registers. Beccause of this, we end up in a situation where -// we have 2 instructions defining the same registers. -static MachineInstr *getOtherVRegDef(const MachineRegisterInfo &MRI, - Register Reg, - const MachineInstr &MI) { - // Is there some way we can assert that there are exactly 2 def instructions? - for (MachineInstr &Other : MRI.def_instructions(Reg)) { - if (&Other != &MI) - return &Other; - } +/// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the +/// rest will be in the remainder. +static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) { + unsigned TotalSize = Ty.getSizeInBits(); + if (!Ty.isVector()) + return {LLT::scalar(FirstSize), LLT::scalar(TotalSize - FirstSize)}; + + LLT EltTy = Ty.getElementType(); + unsigned EltSize = EltTy.getSizeInBits(); + assert(FirstSize % EltSize == 0); - return nullptr; + unsigned FirstPartNumElts = FirstSize / EltSize; + unsigned RemainderElts = (TotalSize - FirstSize) / EltSize; + + return {LLT::scalarOrVector(FirstPartNumElts, EltTy), + LLT::scalarOrVector(RemainderElts, EltTy)}; } -bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI, +static LLT widen96To128(LLT Ty) { + if (!Ty.isVector()) + return LLT::scalar(128); + + LLT EltTy = Ty.getElementType(); + assert(128 % EltTy.getSizeInBits() == 0); + return LLT::vector(128 / EltTy.getSizeInBits(), EltTy); +} + +bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI, const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, MachineRegisterInfo &MRI) const { Register DstReg = MI.getOperand(0).getReg(); - const LLT LoadTy = MRI.getType(DstReg); + const LLT LoadTy = MRI.getType(DstReg); unsigned LoadSize = LoadTy.getSizeInBits(); const unsigned MaxNonSmrdLoadSize = 128; + + const RegisterBank *PtrBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + if (PtrBank == &AMDGPU::SGPRRegBank) { + // If the pointer is an SGPR, we ordinarily have nothing to do. + if (LoadSize != 96) + return false; + + MachineMemOperand *MMO = *MI.memoperands_begin(); + Register PtrReg = MI.getOperand(1).getReg(); + // 96-bit loads are only available for vector loads. We need to split this + // into a 64-bit part, and 32 (unless we can widen to a 128-bit load). + + MachineIRBuilder B(MI); + ApplyRegBankMapping O(*this, MRI, &AMDGPU::SGPRRegBank); + GISelObserverWrapper Observer(&O); + B.setChangeObserver(Observer); + + if (MMO->getAlign() < Align(16)) { + LLT Part64, Part32; + std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64); + auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0); + auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8); + + auto Undef = B.buildUndef(LoadTy); + auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0); + B.buildInsert(MI.getOperand(0), Ins0, Load1, 64); + } else { + LLT WiderTy = widen96To128(LoadTy); + auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0); + B.buildExtract(MI.getOperand(0), WideLoad, 0); + } + + MI.eraseFromParent(); + return true; + } + // 128-bit loads are supported for all instruction types. if (LoadSize <= MaxNonSmrdLoadSize) return false; - SmallVector<unsigned, 16> DefRegs(OpdMapper.getVRegs(0)); - SmallVector<unsigned, 1> SrcRegs(OpdMapper.getVRegs(1)); + SmallVector<Register, 16> DefRegs(OpdMapper.getVRegs(0)); + SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(1)); - // If the pointer is an SGPR, we have nothing to do. - if (SrcRegs.empty()) { - const RegisterBank *PtrBank = - OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; - if (PtrBank == &AMDGPU::SGPRRegBank) - return false; + if (SrcRegs.empty()) SrcRegs.push_back(MI.getOperand(1).getReg()); - } assert(LoadSize % MaxNonSmrdLoadSize == 0); - // We want to get the repair instruction now, because it will help us - // determine which instruction the legalizer inserts that will also - // write to DstReg. - MachineInstr *RepairInst = getOtherVRegDef(MRI, DstReg, MI); - // RegBankSelect only emits scalar types, so we need to reset the pointer // operand to a pointer type. Register BasePtrReg = SrcRegs[0]; @@ -1148,38 +1193,72 @@ bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI, MachineIRBuilder B(MI); - unsigned SplitElts = - MaxNonSmrdLoadSize / LoadTy.getScalarType().getSizeInBits(); - const LLT LoadSplitTy = LLT::vector(SplitElts, LoadTy.getScalarType()); + unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize; + const LLT LoadSplitTy = LoadTy.divide(NumSplitParts); ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank); GISelObserverWrapper Observer(&O); B.setChangeObserver(Observer); LegalizerHelper Helper(B.getMF(), Observer, B); - if (Helper.fewerElementsVector(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized) + + if (LoadTy.isVector()) { + if (Helper.fewerElementsVector(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized) + return false; + } else { + if (Helper.narrowScalar(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized) + return false; + } + + MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); + return true; +} + +bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc( + MachineInstr &MI, + const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, + MachineRegisterInfo &MRI) const { + const MachineFunction &MF = *MI.getMF(); + const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); + const auto &TFI = *ST.getFrameLowering(); + + // Guard in case the stack growth direction ever changes with scratch + // instructions. + if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown) + return false; + + Register Dst = MI.getOperand(0).getReg(); + Register AllocSize = MI.getOperand(1).getReg(); + Align Alignment = assumeAligned(MI.getOperand(2).getImm()); + + const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI); + + // TODO: Need to emit a wave reduction to get the maximum size. + if (SizeBank != &AMDGPU::SGPRRegBank) return false; - // At this point, the legalizer has split the original load into smaller - // loads. At the end of lowering, it inserts an instruction (LegalizedInst) - // that combines the outputs of the lower loads and writes it to DstReg. - // The register bank selector has also added the RepairInst which writes to - // DstReg as well. + LLT PtrTy = MRI.getType(Dst); + LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); + + const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + Register SPReg = Info->getStackPtrOffsetReg(); + ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank); + GISelObserverWrapper Observer(&ApplyBank); - MachineInstr *LegalizedInst = getOtherVRegDef(MRI, DstReg, *RepairInst); + MachineIRBuilder B(MI); + B.setChangeObserver(Observer); - // Replace the output of the LegalizedInst with a temporary register, since - // RepairInst already defines DstReg. - Register TmpReg = MRI.createGenericVirtualRegister(MRI.getType(DstReg)); - LegalizedInst->getOperand(0).setReg(TmpReg); - B.setInsertPt(*RepairInst->getParent(), RepairInst); + auto WaveSize = B.buildConstant(LLT::scalar(32), ST.getWavefrontSizeLog2()); + auto ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize); - for (unsigned DefIdx = 0, e = DefRegs.size(); DefIdx != e; ++DefIdx) { - Register IdxReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); - B.buildConstant(IdxReg, DefIdx); - MRI.setRegBank(IdxReg, AMDGPU::VGPRRegBank); - B.buildExtractVectorElement(DefRegs[DefIdx], TmpReg, IdxReg); + auto SPCopy = B.buildCopy(PtrTy, SPReg); + if (Alignment > TFI.getStackAlign()) { + auto PtrAdd = B.buildPtrAdd(PtrTy, SPCopy, ScaledSize); + B.buildMaskLowPtrBits(Dst, PtrAdd, + Log2(Alignment) + ST.getWavefrontSizeLog2()); + } else { + B.buildPtrAdd(Dst, SPCopy, ScaledSize); } - MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); + MI.eraseFromParent(); return true; } @@ -1210,6 +1289,281 @@ bool AMDGPURegisterBankInfo::applyMappingImage( return true; } +static Register getSrcRegIgnoringCopies(const MachineRegisterInfo &MRI, + Register Reg) { + MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); + if (!Def) + return Reg; + + // TODO: Guard against this being an implicit def + return Def->getOperand(0).getReg(); +} + +// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store +// the three offsets (voffset, soffset and instoffset) +static unsigned setBufferOffsets(MachineIRBuilder &B, + const AMDGPURegisterBankInfo &RBI, + Register CombinedOffset, Register &VOffsetReg, + Register &SOffsetReg, int64_t &InstOffsetVal, + Align Alignment) { + const LLT S32 = LLT::scalar(32); + MachineRegisterInfo *MRI = B.getMRI(); + + if (Optional<int64_t> Imm = getConstantVRegVal(CombinedOffset, *MRI)) { + uint32_t SOffset, ImmOffset; + if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset, &RBI.Subtarget, + Alignment)) { + VOffsetReg = B.buildConstant(S32, 0).getReg(0); + SOffsetReg = B.buildConstant(S32, SOffset).getReg(0); + InstOffsetVal = ImmOffset; + + B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); + B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); + return SOffset + ImmOffset; + } + } + + Register Base; + unsigned Offset; + MachineInstr *Unused; + + std::tie(Base, Offset, Unused) + = AMDGPU::getBaseWithConstantOffset(*MRI, CombinedOffset); + + uint32_t SOffset, ImmOffset; + if (Offset > 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset, + &RBI.Subtarget, Alignment)) { + if (RBI.getRegBank(Base, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) { + VOffsetReg = Base; + SOffsetReg = B.buildConstant(S32, SOffset).getReg(0); + B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); + InstOffsetVal = ImmOffset; + return 0; // XXX - Why is this 0? + } + + // If we have SGPR base, we can use it for soffset. + if (SOffset == 0) { + VOffsetReg = B.buildConstant(S32, 0).getReg(0); + B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); + SOffsetReg = Base; + InstOffsetVal = ImmOffset; + return 0; // XXX - Why is this 0? + } + } + + // Handle the variable sgpr + vgpr case. + if (MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, *MRI)) { + Register Src0 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(1).getReg()); + Register Src1 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(2).getReg()); + + const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, *RBI.TRI); + const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, *RBI.TRI); + + if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) { + VOffsetReg = Src0; + SOffsetReg = Src1; + return 0; + } + + if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) { + VOffsetReg = Src1; + SOffsetReg = Src0; + return 0; + } + } + + // Ensure we have a VGPR for the combined offset. This could be an issue if we + // have an SGPR offset and a VGPR resource. + if (RBI.getRegBank(CombinedOffset, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) { + VOffsetReg = CombinedOffset; + } else { + VOffsetReg = B.buildCopy(S32, CombinedOffset).getReg(0); + B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); + } + + SOffsetReg = B.buildConstant(S32, 0).getReg(0); + B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); + return 0; +} + +bool AMDGPURegisterBankInfo::applyMappingSBufferLoad( + const OperandsMapper &OpdMapper) const { + MachineInstr &MI = OpdMapper.getMI(); + MachineRegisterInfo &MRI = OpdMapper.getMRI(); + + const LLT S32 = LLT::scalar(32); + Register Dst = MI.getOperand(0).getReg(); + LLT Ty = MRI.getType(Dst); + + const RegisterBank *RSrcBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + const RegisterBank *OffsetBank = + OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; + if (RSrcBank == &AMDGPU::SGPRRegBank && + OffsetBank == &AMDGPU::SGPRRegBank) + return true; // Legal mapping + + // FIXME: 96-bit case was widened during legalize. We neeed to narrow it back + // here but don't have an MMO. + + unsigned LoadSize = Ty.getSizeInBits(); + int NumLoads = 1; + if (LoadSize == 256 || LoadSize == 512) { + NumLoads = LoadSize / 128; + Ty = Ty.divide(NumLoads); + } + + // Use the alignment to ensure that the required offsets will fit into the + // immediate offsets. + const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1); + + MachineIRBuilder B(MI); + MachineFunction &MF = B.getMF(); + + Register SOffset; + Register VOffset; + int64_t ImmOffset = 0; + + unsigned MMOOffset = setBufferOffsets(B, *this, MI.getOperand(2).getReg(), + VOffset, SOffset, ImmOffset, Alignment); + + // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we + // can, but we neeed to track an MMO for that. + const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8; + const Align MemAlign(4); // FIXME: ABI type alignment? + MachineMemOperand *BaseMMO = MF.getMachineMemOperand( + MachinePointerInfo(), + MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant, + MemSize, MemAlign); + if (MMOOffset != 0) + BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset, MemSize); + + // If only the offset is divergent, emit a MUBUF buffer load instead. We can + // assume that the buffer is unswizzled. + + Register RSrc = MI.getOperand(1).getReg(); + Register VIndex = B.buildConstant(S32, 0).getReg(0); + B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank); + + SmallVector<Register, 4> LoadParts(NumLoads); + + MachineBasicBlock::iterator MII = MI.getIterator(); + MachineInstrSpan Span(MII, &B.getMBB()); + + for (int i = 0; i < NumLoads; ++i) { + if (NumLoads == 1) { + LoadParts[i] = Dst; + } else { + LoadParts[i] = MRI.createGenericVirtualRegister(Ty); + MRI.setRegBank(LoadParts[i], AMDGPU::VGPRRegBank); + } + + MachineMemOperand *MMO = BaseMMO; + if (i != 0) + BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset + 16 * i, MemSize); + + B.buildInstr(AMDGPU::G_AMDGPU_BUFFER_LOAD) + .addDef(LoadParts[i]) // vdata + .addUse(RSrc) // rsrc + .addUse(VIndex) // vindex + .addUse(VOffset) // voffset + .addUse(SOffset) // soffset + .addImm(ImmOffset + 16 * i) // offset(imm) + .addImm(0) // cachepolicy, swizzled buffer(imm) + .addImm(0) // idxen(imm) + .addMemOperand(MMO); + } + + // TODO: If only the resource is a VGPR, it may be better to execute the + // scalar load in the waterfall loop if the resource is expected to frequently + // be dynamically uniform. + if (RSrcBank != &AMDGPU::SGPRRegBank) { + // Remove the original instruction to avoid potentially confusing the + // waterfall loop logic. + B.setInstr(*Span.begin()); + MI.eraseFromParent(); + + SmallSet<Register, 4> OpsToWaterfall; + + OpsToWaterfall.insert(RSrc); + executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), + OpsToWaterfall, MRI); + } + + if (NumLoads != 1) { + if (Ty.isVector()) + B.buildConcatVectors(Dst, LoadParts); + else + B.buildMerge(Dst, LoadParts); + } + + // We removed the instruction earlier with a waterfall loop. + if (RSrcBank == &AMDGPU::SGPRRegBank) + MI.eraseFromParent(); + + return true; +} + +bool AMDGPURegisterBankInfo::applyMappingBFEIntrinsic( + const OperandsMapper &OpdMapper, bool Signed) const { + MachineInstr &MI = OpdMapper.getMI(); + MachineRegisterInfo &MRI = OpdMapper.getMRI(); + + // Insert basic copies + applyDefaultMapping(OpdMapper); + + Register DstReg = MI.getOperand(0).getReg(); + LLT Ty = MRI.getType(DstReg); + + const LLT S32 = LLT::scalar(32); + + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + if (DstBank == &AMDGPU::VGPRRegBank) { + if (Ty == S32) + return true; + + // TODO: 64-bit version is scalar only, so we need to expand this. + return false; + } + + Register SrcReg = MI.getOperand(2).getReg(); + Register OffsetReg = MI.getOperand(3).getReg(); + Register WidthReg = MI.getOperand(4).getReg(); + + // The scalar form packs the offset and width in a single operand. + + ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank); + GISelObserverWrapper Observer(&ApplyBank); + MachineIRBuilder B(MI); + B.setChangeObserver(Observer); + + // Ensure the high bits are clear to insert the offset. + auto OffsetMask = B.buildConstant(S32, maskTrailingOnes<unsigned>(6)); + auto ClampOffset = B.buildAnd(S32, OffsetReg, OffsetMask); + + // Zeros out the low bits, so don't bother clamping the input value. + auto ShiftWidth = B.buildShl(S32, WidthReg, B.buildConstant(S32, 16)); + + // Transformation function, pack the offset and width of a BFE into + // the format expected by the S_BFE_I32 / S_BFE_U32. In the second + // source, bits [5:0] contain the offset and bits [22:16] the width. + auto MergedInputs = B.buildOr(S32, ClampOffset, ShiftWidth); + + // TODO: It might be worth using a pseudo here to avoid scc clobber and + // register class constraints. + unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) : + (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64); + + auto MIB = B.buildInstr(Opc, {DstReg}, {SrcReg, MergedInputs}); + if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this)) + llvm_unreachable("failed to constrain BFE"); + + MI.eraseFromParent(); + return true; +} + // FIXME: Duplicated from LegalizerHelper static CmpInst::Predicate minMaxToCompare(unsigned Opc) { switch (Opc) { @@ -1226,6 +1580,51 @@ static CmpInst::Predicate minMaxToCompare(unsigned Opc) { } } +static unsigned minMaxToExtend(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + return TargetOpcode::G_SEXT; + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: + return TargetOpcode::G_ZEXT; + default: + llvm_unreachable("not in integer min/max"); + } +} + +// Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding +// any illegal vector extend or unmerge operations. +static std::pair<Register, Register> +unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) { + const LLT S32 = LLT::scalar(32); + auto Bitcast = B.buildBitcast(S32, Src); + + if (ExtOpcode == TargetOpcode::G_SEXT) { + auto ExtLo = B.buildSExtInReg(S32, Bitcast, 16); + auto ShiftHi = B.buildAShr(S32, Bitcast, B.buildConstant(S32, 16)); + return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0)); + } + + auto ShiftHi = B.buildLShr(S32, Bitcast, B.buildConstant(S32, 16)); + if (ExtOpcode == TargetOpcode::G_ZEXT) { + auto ExtLo = B.buildAnd(S32, Bitcast, B.buildConstant(S32, 0xffff)); + return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0)); + } + + assert(ExtOpcode == TargetOpcode::G_ANYEXT); + return std::make_pair(Bitcast.getReg(0), ShiftHi.getReg(0)); +} + +static MachineInstr *buildExpandedScalarMinMax(MachineIRBuilder &B, + CmpInst::Predicate Pred, + Register Dst, Register Src0, + Register Src1) { + const LLT CmpType = LLT::scalar(32); + auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1); + return B.buildSelect(Dst, Cmp, Src0, Src1); +} + // FIXME: Duplicated from LegalizerHelper, except changing the boolean type. void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B, MachineInstr &MI) const { @@ -1234,24 +1633,25 @@ void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B, Register Src1 = MI.getOperand(2).getReg(); const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); - LLT CmpType = LLT::scalar(32); - - auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1); - B.buildSelect(Dst, Cmp, Src0, Src1); + MachineInstr *Sel = buildExpandedScalarMinMax(B, Pred, Dst, Src0, Src1); - B.getMRI()->setRegBank(Cmp.getReg(0), AMDGPU::SGPRRegBank); + Register CmpReg = Sel->getOperand(1).getReg(); + B.getMRI()->setRegBank(CmpReg, AMDGPU::SGPRRegBank); MI.eraseFromParent(); } // For cases where only a single copy is inserted for matching register banks. // Replace the register in the instruction operand -static void substituteSimpleCopyRegs( +static bool substituteSimpleCopyRegs( const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) { SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx)); if (!SrcReg.empty()) { assert(SrcReg.size() == 1); OpdMapper.getMI().getOperand(OpIdx).setReg(SrcReg[0]); + return true; } + + return false; } /// Handle register layout difference for f16 images for some subtargets. @@ -1465,6 +1865,223 @@ bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg, constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI); } +/// Utility function for pushing dynamic vector indexes with a constant offset +/// into waterwall loops. +static void reinsertVectorIndexAdd(MachineIRBuilder &B, + MachineInstr &IdxUseInstr, + unsigned OpIdx, + unsigned ConstOffset) { + MachineRegisterInfo &MRI = *B.getMRI(); + const LLT S32 = LLT::scalar(32); + Register WaterfallIdx = IdxUseInstr.getOperand(OpIdx).getReg(); + B.setInsertPt(*IdxUseInstr.getParent(), IdxUseInstr.getIterator()); + + auto MaterializedOffset = B.buildConstant(S32, ConstOffset); + + auto Add = B.buildAdd(S32, WaterfallIdx, MaterializedOffset); + MRI.setRegBank(MaterializedOffset.getReg(0), AMDGPU::SGPRRegBank); + MRI.setRegBank(Add.getReg(0), AMDGPU::SGPRRegBank); + IdxUseInstr.getOperand(OpIdx).setReg(Add.getReg(0)); +} + +/// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the +/// original 32-bit source value (to be inserted in the low part of the combined +/// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit +/// value. +static void extendLow32IntoHigh32(MachineIRBuilder &B, + Register Hi32Reg, Register Lo32Reg, + unsigned ExtOpc, + const RegisterBank &RegBank, + bool IsBooleanSrc = false) { + if (ExtOpc == AMDGPU::G_ZEXT) { + B.buildConstant(Hi32Reg, 0); + } else if (ExtOpc == AMDGPU::G_SEXT) { + if (IsBooleanSrc) { + // If we know the original source was an s1, the high half is the same as + // the low. + B.buildCopy(Hi32Reg, Lo32Reg); + } else { + // Replicate sign bit from 32-bit extended part. + auto ShiftAmt = B.buildConstant(LLT::scalar(32), 31); + B.getMRI()->setRegBank(ShiftAmt.getReg(0), RegBank); + B.buildAShr(Hi32Reg, Lo32Reg, ShiftAmt); + } + } else { + assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension"); + B.buildUndef(Hi32Reg); + } +} + +bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect( + MachineInstr &MI, MachineRegisterInfo &MRI, + const OperandsMapper &OpdMapper) const { + + Register VecReg = MI.getOperand(1).getReg(); + Register Idx = MI.getOperand(2).getReg(); + + const RegisterBank &IdxBank = + *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; + + bool IsDivergentIdx = IdxBank == AMDGPU::VGPRRegBank; + + LLT VecTy = MRI.getType(VecReg); + unsigned EltSize = VecTy.getScalarSizeInBits(); + unsigned NumElem = VecTy.getNumElements(); + + if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem, + IsDivergentIdx)) + return false; + + MachineIRBuilder B(MI); + LLT S32 = LLT::scalar(32); + + const RegisterBank &DstBank = + *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + const RegisterBank &SrcBank = + *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + + const RegisterBank &CCBank = + (DstBank == AMDGPU::SGPRRegBank && + SrcBank == AMDGPU::SGPRRegBank && + IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank + : AMDGPU::VCCRegBank; + LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); + + if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { + Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg(); + MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); + } + + LLT EltTy = VecTy.getScalarType(); + SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0)); + unsigned NumLanes = DstRegs.size(); + if (!NumLanes) + NumLanes = 1; + else + EltTy = MRI.getType(DstRegs[0]); + + auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg); + SmallVector<Register, 2> Res(NumLanes); + for (unsigned L = 0; L < NumLanes; ++L) + Res[L] = UnmergeToEltTy.getReg(L); + + for (unsigned I = 1; I < NumElem; ++I) { + auto IC = B.buildConstant(S32, I); + MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); + auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC); + MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank); + + for (unsigned L = 0; L < NumLanes; ++L) { + auto S = B.buildSelect(EltTy, Cmp, + UnmergeToEltTy.getReg(I * NumLanes + L), Res[L]); + + for (unsigned N : { 0, 2, 3 }) + MRI.setRegBank(S->getOperand(N).getReg(), DstBank); + + Res[L] = S->getOperand(0).getReg(); + } + } + + for (unsigned L = 0; L < NumLanes; ++L) { + Register DstReg = (NumLanes == 1) ? MI.getOperand(0).getReg() : DstRegs[L]; + B.buildCopy(DstReg, Res[L]); + MRI.setRegBank(DstReg, DstBank); + } + + MRI.setRegBank(MI.getOperand(0).getReg(), DstBank); + MI.eraseFromParent(); + + return true; +} + +bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect( + MachineInstr &MI, MachineRegisterInfo &MRI, + const OperandsMapper &OpdMapper) const { + + Register VecReg = MI.getOperand(1).getReg(); + Register Idx = MI.getOperand(3).getReg(); + + const RegisterBank &IdxBank = + *OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank; + + bool IsDivergentIdx = IdxBank == AMDGPU::VGPRRegBank; + + LLT VecTy = MRI.getType(VecReg); + unsigned EltSize = VecTy.getScalarSizeInBits(); + unsigned NumElem = VecTy.getNumElements(); + + if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem, + IsDivergentIdx)) + return false; + + MachineIRBuilder B(MI); + LLT S32 = LLT::scalar(32); + + const RegisterBank &DstBank = + *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + const RegisterBank &SrcBank = + *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + const RegisterBank &InsBank = + *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; + + const RegisterBank &CCBank = + (DstBank == AMDGPU::SGPRRegBank && + SrcBank == AMDGPU::SGPRRegBank && + InsBank == AMDGPU::SGPRRegBank && + IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank + : AMDGPU::VCCRegBank; + LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); + + if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { + Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg(); + MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); + } + + LLT EltTy = VecTy.getScalarType(); + SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2)); + unsigned NumLanes = InsRegs.size(); + if (!NumLanes) { + NumLanes = 1; + InsRegs.push_back(MI.getOperand(2).getReg()); + } else { + EltTy = MRI.getType(InsRegs[0]); + } + + auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg); + SmallVector<Register, 16> Ops(NumElem * NumLanes); + + for (unsigned I = 0; I < NumElem; ++I) { + auto IC = B.buildConstant(S32, I); + MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); + auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC); + MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank); + + for (unsigned L = 0; L < NumLanes; ++L) { + auto S = B.buildSelect(EltTy, Cmp, InsRegs[L], + UnmergeToEltTy.getReg(I * NumLanes + L)); + + for (unsigned N : { 0, 2, 3 }) + MRI.setRegBank(S->getOperand(N).getReg(), DstBank); + + Ops[I * NumLanes + L] = S->getOperand(0).getReg(); + } + } + + LLT MergeTy = LLT::vector(Ops.size(), EltTy); + if (MergeTy == MRI.getType(MI.getOperand(0).getReg())) { + B.buildBuildVector(MI.getOperand(0), Ops); + } else { + auto Vec = B.buildBuildVector(MergeTy, Ops); + MRI.setRegBank(Vec->getOperand(0).getReg(), DstBank); + B.buildBitcast(MI.getOperand(0).getReg(), Vec); + } + + MRI.setRegBank(MI.getOperand(0).getReg(), DstBank); + MI.eraseFromParent(); + + return true; +} + void AMDGPURegisterBankInfo::applyMappingImpl( const OperandsMapper &OpdMapper) const { MachineInstr &MI = OpdMapper.getMI(); @@ -1555,7 +2172,13 @@ void AMDGPURegisterBankInfo::applyMappingImpl( MachineBasicBlock *MBB = MI.getParent(); B.setInsertPt(*MBB, std::next(MI.getIterator())); - B.buildTrunc(DstReg, NewDstReg); + + // If we had a constrained VCC result register, a copy was inserted to VCC + // from SGPR. + SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(0)); + if (DefRegs.empty()) + DefRegs.push_back(DstReg); + B.buildTrunc(DefRegs[0], NewDstReg); return; } case AMDGPU::G_SELECT: { @@ -1712,10 +2335,16 @@ void AMDGPURegisterBankInfo::applyMappingImpl( } case AMDGPU::G_ADD: case AMDGPU::G_SUB: - case AMDGPU::G_MUL: { + case AMDGPU::G_MUL: + case AMDGPU::G_SHL: + case AMDGPU::G_LSHR: + case AMDGPU::G_ASHR: { Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (DstTy != LLT::scalar(16)) + + // 16-bit operations are VALU only, but can be promoted to 32-bit SALU. + // Packed 16-bit operations need to be scalarized and promoted. + if (DstTy != LLT::scalar(16) && DstTy != LLT::vector(2, 16)) break; const RegisterBank *DstBank = @@ -1723,16 +2352,42 @@ void AMDGPURegisterBankInfo::applyMappingImpl( if (DstBank == &AMDGPU::VGPRRegBank) break; - // 16-bit operations are VALU only, but can be promoted to 32-bit SALU. - MachineFunction *MF = MI.getParent()->getParent(); + const LLT S32 = LLT::scalar(32); + MachineBasicBlock *MBB = MI.getParent(); + MachineFunction *MF = MBB->getParent(); MachineIRBuilder B(MI); ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); GISelObserverWrapper Observer(&ApplySALU); - LegalizerHelper Helper(*MF, Observer, B); - if (Helper.widenScalar(MI, 0, LLT::scalar(32)) != - LegalizerHelper::Legalized) - llvm_unreachable("widen scalar should have succeeded"); + if (DstTy.isVector()) { + B.setChangeObserver(Observer); + + Register WideSrc0Lo, WideSrc0Hi; + Register WideSrc1Lo, WideSrc1Hi; + + std::tie(WideSrc0Lo, WideSrc0Hi) + = unpackV2S16ToS32(B, MI.getOperand(1).getReg(), AMDGPU::G_ANYEXT); + std::tie(WideSrc1Lo, WideSrc1Hi) + = unpackV2S16ToS32(B, MI.getOperand(2).getReg(), AMDGPU::G_ANYEXT); + auto Lo = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Lo, WideSrc1Lo}); + auto Hi = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Hi, WideSrc1Hi}); + B.buildBuildVectorTrunc(DstReg, {Lo.getReg(0), Hi.getReg(0)}); + MI.eraseFromParent(); + } else { + LegalizerHelper Helper(*MF, Observer, B); + + if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) + llvm_unreachable("widen scalar should have succeeded"); + + // FIXME: s16 shift amounts should be legal. + if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR || + Opc == AMDGPU::G_ASHR) { + B.setInsertPt(*MBB, MI.getIterator()); + if (Helper.widenScalar(MI, 1, S32) != LegalizerHelper::Legalized) + llvm_unreachable("widen scalar should have succeeded"); + } + } + return; } case AMDGPU::G_SMIN: @@ -1750,10 +2405,44 @@ void AMDGPURegisterBankInfo::applyMappingImpl( // Turn scalar min/max into a compare and select. LLT Ty = MRI.getType(DstReg); - LLT S32 = LLT::scalar(32); - LLT S16 = LLT::scalar(16); + const LLT S32 = LLT::scalar(32); + const LLT S16 = LLT::scalar(16); + const LLT V2S16 = LLT::vector(2, 16); + + if (Ty == V2S16) { + ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); + GISelObserverWrapper Observer(&ApplySALU); + B.setChangeObserver(Observer); - if (Ty == S16) { + // Need to widen to s32, and expand as cmp + select, and avoid producing + // illegal vector extends or unmerges that would need further + // legalization. + // + // TODO: Should we just readfirstlane? That should probably be handled + // with a UniformVGPR register bank that wouldn't need special + // consideration here. + + Register Dst = MI.getOperand(0).getReg(); + Register Src0 = MI.getOperand(1).getReg(); + Register Src1 = MI.getOperand(2).getReg(); + + Register WideSrc0Lo, WideSrc0Hi; + Register WideSrc1Lo, WideSrc1Hi; + + unsigned ExtendOp = minMaxToExtend(MI.getOpcode()); + + std::tie(WideSrc0Lo, WideSrc0Hi) = unpackV2S16ToS32(B, Src0, ExtendOp); + std::tie(WideSrc1Lo, WideSrc1Hi) = unpackV2S16ToS32(B, Src1, ExtendOp); + + Register Lo = MRI.createGenericVirtualRegister(S32); + Register Hi = MRI.createGenericVirtualRegister(S32); + const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); + buildExpandedScalarMinMax(B, Pred, Lo, WideSrc0Lo, WideSrc1Lo); + buildExpandedScalarMinMax(B, Pred, Hi, WideSrc0Hi, WideSrc1Hi); + + B.buildBuildVectorTrunc(Dst, {Lo, Hi}); + MI.eraseFromParent(); + } else if (Ty == S16) { ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); GISelObserverWrapper Observer(&ApplySALU); LegalizerHelper Helper(*MF, Observer, B); @@ -1769,11 +2458,77 @@ void AMDGPURegisterBankInfo::applyMappingImpl( return; } + case AMDGPU::G_SEXT_INREG: { + SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1)); + if (SrcRegs.empty()) + break; // Nothing to repair + + const LLT S32 = LLT::scalar(32); + MachineIRBuilder B(MI); + ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank); + GISelObserverWrapper Observer(&O); + B.setChangeObserver(Observer); + + // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs + // we would need to further expand, and doesn't let us directly set the + // result registers. + SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0)); + + int Amt = MI.getOperand(2).getImm(); + if (Amt <= 32) { + if (Amt == 32) { + // The low bits are unchanged. + B.buildCopy(DstRegs[0], SrcRegs[0]); + } else { + // Extend in the low bits and propagate the sign bit to the high half. + B.buildSExtInReg(DstRegs[0], SrcRegs[0], Amt); + } + + B.buildAShr(DstRegs[1], DstRegs[0], B.buildConstant(S32, 31)); + } else { + // The low bits are unchanged, and extend in the high bits. + B.buildCopy(DstRegs[0], SrcRegs[0]); + B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32); + } + + Register DstReg = MI.getOperand(0).getReg(); + MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); + MI.eraseFromParent(); + return; + } + case AMDGPU::G_CTPOP: + case AMDGPU::G_CTLZ_ZERO_UNDEF: + case AMDGPU::G_CTTZ_ZERO_UNDEF: { + MachineIRBuilder B(MI); + MachineFunction &MF = B.getMF(); + + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + if (DstBank == &AMDGPU::SGPRRegBank) + break; + + Register SrcReg = MI.getOperand(1).getReg(); + const LLT S32 = LLT::scalar(32); + LLT Ty = MRI.getType(SrcReg); + if (Ty == S32) + break; + + ApplyRegBankMapping ApplyVALU(*this, MRI, &AMDGPU::VGPRRegBank); + GISelObserverWrapper Observer(&ApplyVALU); + LegalizerHelper Helper(MF, Observer, B); + + if (Helper.narrowScalar(MI, 1, S32) != LegalizerHelper::Legalized) + llvm_unreachable("narrowScalar should have succeeded"); + return; + } case AMDGPU::G_SEXT: - case AMDGPU::G_ZEXT: { + case AMDGPU::G_ZEXT: + case AMDGPU::G_ANYEXT: { Register SrcReg = MI.getOperand(1).getReg(); LLT SrcTy = MRI.getType(SrcReg); - bool Signed = Opc == AMDGPU::G_SEXT; + const bool Signed = Opc == AMDGPU::G_SEXT; + + assert(empty(OpdMapper.getVRegs(1))); MachineIRBuilder B(MI); const RegisterBank *SrcBank = @@ -1788,23 +2543,19 @@ void AMDGPURegisterBankInfo::applyMappingImpl( // breakdowns supported. DstTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() <= 32) { - const LLT S32 = LLT::scalar(32); SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); // Extend to 32-bit, and then extend the low half. if (Signed) { // TODO: Should really be buildSExtOrCopy B.buildSExtOrTrunc(DefRegs[0], SrcReg); - - // Replicate sign bit from 32-bit extended part. - auto ShiftAmt = B.buildConstant(S32, 31); - MRI.setRegBank(ShiftAmt.getReg(0), *SrcBank); - B.buildAShr(DefRegs[1], DefRegs[0], ShiftAmt); - } else { + } else if (Opc == AMDGPU::G_ZEXT) { B.buildZExtOrTrunc(DefRegs[0], SrcReg); - B.buildConstant(DefRegs[1], 0); + } else { + B.buildAnyExtOrTrunc(DefRegs[0], SrcReg); } + extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank); MRI.setRegBank(DstReg, *SrcBank); MI.eraseFromParent(); return; @@ -1813,6 +2564,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl( if (SrcTy != LLT::scalar(1)) return; + // It is not legal to have a legalization artifact with a VCC source. Rather + // than introducing a copy, insert the select we would have to select the + // copy to. if (SrcBank == &AMDGPU::VCCRegBank) { SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); @@ -1834,7 +2588,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl( if (DstSize > 32) { B.buildSelect(DefRegs[0], SrcReg, True, False); - B.buildCopy(DefRegs[1], DefRegs[0]); + extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank, true); } else if (DstSize < 32) { auto Sel = B.buildSelect(SelType, SrcReg, True, False); MRI.setRegBank(Sel.getReg(0), *DstBank); @@ -1847,24 +2601,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl( return; } - // Fixup the case with an s1 src that isn't a condition register. Use shifts - // instead of introducing a compare to avoid an unnecessary condition - // register (and since there's no scalar 16-bit compares). - auto Ext = B.buildAnyExt(DstTy, SrcReg); - auto ShiftAmt = B.buildConstant(LLT::scalar(32), DstTy.getSizeInBits() - 1); - auto Shl = B.buildShl(DstTy, Ext, ShiftAmt); - - if (MI.getOpcode() == AMDGPU::G_SEXT) - B.buildAShr(DstReg, Shl, ShiftAmt); - else - B.buildLShr(DstReg, Shl, ShiftAmt); - - MRI.setRegBank(DstReg, *SrcBank); - MRI.setRegBank(Ext.getReg(0), *SrcBank); - MRI.setRegBank(ShiftAmt.getReg(0), *SrcBank); - MRI.setRegBank(Shl.getReg(0), *SrcBank); - MI.eraseFromParent(); - return; + break; } case AMDGPU::G_BUILD_VECTOR: case AMDGPU::G_BUILD_VECTOR_TRUNC: { @@ -1934,7 +2671,16 @@ void AMDGPURegisterBankInfo::applyMappingImpl( assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty()); - LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + + const LLT S32 = LLT::scalar(32); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(SrcReg); + + if (foldExtractEltToCmpSelect(MI, MRI, OpdMapper)) + return; + MachineIRBuilder B(MI); const ValueMapping &DstMapping @@ -1942,10 +2688,26 @@ void AMDGPURegisterBankInfo::applyMappingImpl( const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank; const RegisterBank *SrcBank = OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + const RegisterBank *IdxBank = + OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; - Register DstReg = MI.getOperand(0).getReg(); - Register SrcReg = MI.getOperand(1).getReg(); - Register IdxReg = MI.getOperand(2).getReg(); + Register BaseIdxReg; + unsigned ConstOffset; + MachineInstr *OffsetDef; + std::tie(BaseIdxReg, ConstOffset, OffsetDef) = + AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg()); + + // See if the index is an add of a constant which will be foldable by moving + // the base register of the index later if this is going to be executed in a + // waterfall loop. This is essentially to reassociate the add of a constant + // with the readfirstlane. + bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && + ConstOffset > 0 && + ConstOffset < SrcTy.getNumElements(); + + // Move the base register. We'll re-insert the add later. + if (ShouldMoveIndexIntoLoop) + MI.getOperand(2).setReg(BaseIdxReg); // If this is a VGPR result only because the index was a VGPR result, the // actual indexing will be done on the SGPR source vector, which will @@ -1969,26 +2731,30 @@ void AMDGPURegisterBankInfo::applyMappingImpl( buildVCopy(B, DstReg, TmpReg); } + // Re-insert the constant offset add inside the waterfall loop. + if (ShouldMoveIndexIntoLoop) + reinsertVectorIndexAdd(B, MI, 2, ConstOffset); + return; } assert(DstTy.getSizeInBits() == 64); - LLT SrcTy = MRI.getType(SrcReg); - const LLT S32 = LLT::scalar(32); LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32); auto CastSrc = B.buildBitcast(Vec32, SrcReg); auto One = B.buildConstant(S32, 1); + MachineBasicBlock::iterator MII = MI.getIterator(); + // Split the vector index into 32-bit pieces. Prepare to move all of the // new instructions into a waterfall loop if necessary. // // Don't put the bitcast or constant in the loop. - MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB()); + MachineInstrSpan Span(MII, &B.getMBB()); // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1). - auto IdxLo = B.buildShl(S32, IdxReg, One); + auto IdxLo = B.buildShl(S32, BaseIdxReg, One); auto IdxHi = B.buildAdd(S32, IdxLo, One); auto Extract0 = B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo); @@ -2029,33 +2795,70 @@ void AMDGPURegisterBankInfo::applyMappingImpl( buildVCopy(B, DstRegs[1], TmpReg1); } + if (ShouldMoveIndexIntoLoop) + reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset); + return; } case AMDGPU::G_INSERT_VECTOR_ELT: { SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2)); + Register DstReg = MI.getOperand(0).getReg(); + LLT VecTy = MRI.getType(DstReg); + assert(OpdMapper.getVRegs(0).empty()); - assert(OpdMapper.getVRegs(1).empty()); assert(OpdMapper.getVRegs(3).empty()); - if (InsRegs.empty()) { - applyDefaultMapping(OpdMapper); - executeInWaterfallLoop(MI, MRI, { 3 }); + if (substituteSimpleCopyRegs(OpdMapper, 1)) + MRI.setType(MI.getOperand(1).getReg(), VecTy); + + if (foldInsertEltToCmpSelect(MI, MRI, OpdMapper)) return; - } - Register DstReg = MI.getOperand(0).getReg(); + const RegisterBank *IdxBank = + OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank; + Register SrcReg = MI.getOperand(1).getReg(); Register InsReg = MI.getOperand(2).getReg(); - Register IdxReg = MI.getOperand(3).getReg(); - LLT SrcTy = MRI.getType(SrcReg); LLT InsTy = MRI.getType(InsReg); (void)InsTy; + Register BaseIdxReg; + unsigned ConstOffset; + MachineInstr *OffsetDef; + std::tie(BaseIdxReg, ConstOffset, OffsetDef) = + AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(3).getReg()); + + // See if the index is an add of a constant which will be foldable by moving + // the base register of the index later if this is going to be executed in a + // waterfall loop. This is essentially to reassociate the add of a constant + // with the readfirstlane. + bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && + ConstOffset > 0 && + ConstOffset < VecTy.getNumElements(); + + // Move the base register. We'll re-insert the add later. + if (ShouldMoveIndexIntoLoop) + MI.getOperand(3).setReg(BaseIdxReg); + + + if (InsRegs.empty()) { + executeInWaterfallLoop(MI, MRI, { 3 }); + + // Re-insert the constant offset add inside the waterfall loop. + if (ShouldMoveIndexIntoLoop) { + MachineIRBuilder B(MI); + reinsertVectorIndexAdd(B, MI, 3, ConstOffset); + } + + return; + } + + assert(InsTy.getSizeInBits() == 64); const LLT S32 = LLT::scalar(32); - LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32); + LLT Vec32 = LLT::vector(2 * VecTy.getNumElements(), 32); MachineIRBuilder B(MI); auto CastSrc = B.buildBitcast(Vec32, SrcReg); @@ -2068,12 +2871,11 @@ void AMDGPURegisterBankInfo::applyMappingImpl( MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB()); // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1). - auto IdxLo = B.buildShl(S32, IdxReg, One); + auto IdxLo = B.buildShl(S32, BaseIdxReg, One); auto IdxHi = B.buildAdd(S32, IdxLo, One); auto InsLo = B.buildInsertVectorElement(Vec32, CastSrc, InsRegs[0], IdxLo); auto InsHi = B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi); - B.buildBitcast(DstReg, InsHi); const RegisterBank *DstBank = OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; @@ -2093,6 +2895,8 @@ void AMDGPURegisterBankInfo::applyMappingImpl( SmallSet<Register, 4> OpsToWaterfall; if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 3 })) { + B.setInsertPt(B.getMBB(), MI); + B.buildBitcast(DstReg, InsHi); MI.eraseFromParent(); return; } @@ -2100,17 +2904,70 @@ void AMDGPURegisterBankInfo::applyMappingImpl( B.setInstr(*Span.begin()); MI.eraseFromParent(); + // Figure out the point after the waterfall loop before mangling the control + // flow. executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), OpsToWaterfall, MRI); + + // The insertion point is now right after the original instruction. + // + // Keep the bitcast to the original vector type out of the loop. Doing this + // saved an extra phi we don't need inside the loop. + B.buildBitcast(DstReg, InsHi); + + // Re-insert the constant offset add inside the waterfall loop. + if (ShouldMoveIndexIntoLoop) + reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset); + + return; + } + case AMDGPU::G_AMDGPU_BUFFER_LOAD: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: + case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: + case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: + case AMDGPU::G_AMDGPU_BUFFER_STORE: + case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: + case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: + case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: + case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: { + applyDefaultMapping(OpdMapper); + executeInWaterfallLoop(MI, MRI, {1, 4}); + return; + } + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: { + applyDefaultMapping(OpdMapper); + executeInWaterfallLoop(MI, MRI, {2, 5}); + return; + } + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { + applyDefaultMapping(OpdMapper); + executeInWaterfallLoop(MI, MRI, {3, 6}); + return; + } + case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: { + applyMappingSBufferLoad(OpdMapper); return; } case AMDGPU::G_INTRINSIC: { switch (MI.getIntrinsicID()) { - case Intrinsic::amdgcn_s_buffer_load: { - // FIXME: Move to G_INTRINSIC_W_SIDE_EFFECTS - executeInWaterfallLoop(MI, MRI, { 2, 3 }); - return; - } case Intrinsic::amdgcn_readlane: { substituteSimpleCopyRegs(OpdMapper, 2); @@ -2132,18 +2989,51 @@ void AMDGPURegisterBankInfo::applyMappingImpl( constrainOpWithReadfirstlane(MI, MRI, 3); // Index return; } - default: - break; + case Intrinsic::amdgcn_ballot: + case Intrinsic::amdgcn_interp_p1: + case Intrinsic::amdgcn_interp_p2: + case Intrinsic::amdgcn_interp_mov: + case Intrinsic::amdgcn_interp_p1_f16: + case Intrinsic::amdgcn_interp_p2_f16: { + applyDefaultMapping(OpdMapper); + + // Readlane for m0 value, which is always the last operand. + // FIXME: Should this be a waterfall loop instead? + constrainOpWithReadfirstlane(MI, MRI, MI.getNumOperands() - 1); // Index + return; + } + case Intrinsic::amdgcn_permlane16: + case Intrinsic::amdgcn_permlanex16: { + // Doing a waterfall loop over these wouldn't make any sense. + substituteSimpleCopyRegs(OpdMapper, 2); + substituteSimpleCopyRegs(OpdMapper, 3); + constrainOpWithReadfirstlane(MI, MRI, 4); + constrainOpWithReadfirstlane(MI, MRI, 5); + return; + } + case Intrinsic::amdgcn_sbfe: + applyMappingBFEIntrinsic(OpdMapper, true); + return; + case Intrinsic::amdgcn_ubfe: + applyMappingBFEIntrinsic(OpdMapper, false); + return; } break; } + case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: + case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: { + const AMDGPU::RsrcIntrinsic *RSrcIntrin + = AMDGPU::lookupRsrcIntrinsic(MI.getIntrinsicID()); + assert(RSrcIntrin && RSrcIntrin->IsImage); + // Non-images can have complications from operands that allow both SGPR + // and VGPR. For now it's too complicated to figure out the final opcode + // to derive the register bank from the MCInstrDesc. + applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg); + return; + } case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: { auto IntrID = MI.getIntrinsicID(); switch (IntrID) { - case Intrinsic::amdgcn_buffer_load: { - executeInWaterfallLoop(MI, MRI, { 2 }); - return; - } case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_swap: { // This is only allowed to execute with 1 lane, so readfirstlane is safe. @@ -2167,28 +3057,19 @@ void AMDGPURegisterBankInfo::applyMappingImpl( constrainOpWithReadfirstlane(MI, MRI, 1); // M0 return; } + case Intrinsic::amdgcn_ds_append: + case Intrinsic::amdgcn_ds_consume: { + constrainOpWithReadfirstlane(MI, MRI, 2); // M0 + return; + } case Intrinsic::amdgcn_s_sendmsg: case Intrinsic::amdgcn_s_sendmsghalt: { // FIXME: Should this use a waterfall loop? constrainOpWithReadfirstlane(MI, MRI, 2); // M0 return; } - case Intrinsic::amdgcn_raw_buffer_load: - case Intrinsic::amdgcn_raw_buffer_load_format: - case Intrinsic::amdgcn_raw_tbuffer_load: - case Intrinsic::amdgcn_raw_buffer_store: - case Intrinsic::amdgcn_raw_buffer_store_format: - case Intrinsic::amdgcn_raw_tbuffer_store: { - applyDefaultMapping(OpdMapper); - executeInWaterfallLoop(MI, MRI, {2, 4}); - return; - } - case Intrinsic::amdgcn_struct_buffer_load: - case Intrinsic::amdgcn_struct_buffer_store: - case Intrinsic::amdgcn_struct_tbuffer_load: - case Intrinsic::amdgcn_struct_tbuffer_store: { - applyDefaultMapping(OpdMapper); - executeInWaterfallLoop(MI, MRI, {2, 5}); + case Intrinsic::amdgcn_s_setreg: { + constrainOpWithReadfirstlane(MI, MRI, 2); return; } default: { @@ -2211,10 +3092,13 @@ void AMDGPURegisterBankInfo::applyMappingImpl( case AMDGPU::G_LOAD: case AMDGPU::G_ZEXTLOAD: case AMDGPU::G_SEXTLOAD: { - if (applyMappingWideLoad(MI, OpdMapper, MRI)) + if (applyMappingLoad(MI, OpdMapper, MRI)) return; break; } + case AMDGPU::G_DYN_STACKALLOC: + applyMappingDynStackAlloc(MI, OpdMapper, MRI); + return; default: break; } @@ -2244,7 +3128,11 @@ AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const { SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { - unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI); + const MachineOperand &SrcOp = MI.getOperand(i); + if (!SrcOp.isReg()) + continue; + + unsigned Size = getSizeInBits(SrcOp.getReg(), MRI, *TRI); OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); } return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), @@ -2256,31 +3144,19 @@ AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const { const MachineFunction &MF = *MI.getParent()->getParent(); const MachineRegisterInfo &MRI = MF.getRegInfo(); SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); - unsigned OpdIdx = 0; - - unsigned Size0 = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); - OpdsMapping[OpdIdx++] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size0); - - if (MI.getOperand(OpdIdx).isIntrinsicID()) - OpdsMapping[OpdIdx++] = nullptr; - - Register Reg1 = MI.getOperand(OpdIdx).getReg(); - unsigned Size1 = getSizeInBits(Reg1, MRI, *TRI); - - unsigned DefaultBankID = Size1 == 1 ? - AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID; - unsigned Bank1 = getRegBankID(Reg1, MRI, *TRI, DefaultBankID); - OpdsMapping[OpdIdx++] = AMDGPU::getValueMapping(Bank1, Size1); - - for (unsigned e = MI.getNumOperands(); OpdIdx != e; ++OpdIdx) { - const MachineOperand &MO = MI.getOperand(OpdIdx); - if (!MO.isReg()) + // Even though we technically could use SGPRs, this would require knowledge of + // the constant bus restriction. Force all sources to VGPR (except for VCC). + // + // TODO: Unary ops are trivially OK, so accept SGPRs? + for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { + const MachineOperand &Src = MI.getOperand(i); + if (!Src.isReg()) continue; - unsigned Size = getSizeInBits(MO.getReg(), MRI, *TRI); + unsigned Size = getSizeInBits(Src.getReg(), MRI, *TRI); unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID; - OpdsMapping[OpdIdx] = AMDGPU::getValueMapping(BankID, Size); + OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size); } return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), @@ -2324,6 +3200,10 @@ AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI, continue; Register OpReg = MI.getOperand(I).getReg(); + // We replace some dead address operands with $noreg + if (!OpReg) + continue; + unsigned Size = getSizeInBits(OpReg, MRI, *TRI); // FIXME: Probably need a new intrinsic register bank searchable table to @@ -2345,6 +3225,22 @@ AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI, return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), NumOps); } +/// Return the mapping for a pointer arugment. +const RegisterBankInfo::ValueMapping * +AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI, + Register PtrReg) const { + LLT PtrTy = MRI.getType(PtrReg); + unsigned Size = PtrTy.getSizeInBits(); + if (Subtarget.useFlatForGlobal() || + !SITargetLowering::isFlatGlobalAddrSpace(PtrTy.getAddressSpace())) + return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); + + // If we're using MUBUF instructions for global memory, an SGPR base register + // is possible. Otherwise this needs to be a VGPR. + const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI); + return AMDGPU::getValueMapping(PtrBank->getID(), Size); +} + const RegisterBankInfo::InstructionMapping & AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const { @@ -2352,7 +3248,6 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const { const MachineRegisterInfo &MRI = MF.getRegInfo(); SmallVector<const ValueMapping*, 2> OpdsMapping(2); unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); - LLT LoadTy = MRI.getType(MI.getOperand(0).getReg()); Register PtrReg = MI.getOperand(1).getReg(); LLT PtrTy = MRI.getType(PtrReg); unsigned AS = PtrTy.getAddressSpace(); @@ -2364,14 +3259,23 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const { const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI); if (PtrBank == &AMDGPU::SGPRRegBank && - (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS && - AS != AMDGPUAS::PRIVATE_ADDRESS) && - isScalarLoadLegal(MI)) { - // We have a uniform instruction so we want to use an SMRD load - ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); - PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize); + SITargetLowering::isFlatGlobalAddrSpace(AS)) { + if (isScalarLoadLegal(MI)) { + // We have a uniform instruction so we want to use an SMRD load + ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); + PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize); + } else { + ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); + + // If we're using MUBUF instructions for global memory, an SGPR base + // register is possible. Otherwise this needs to be a VGPR. + unsigned PtrBankID = Subtarget.useFlatForGlobal() ? + AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID; + + PtrMapping = AMDGPU::getValueMapping(PtrBankID, PtrSize); + } } else { - ValMapping = AMDGPU::getValueMappingLoadSGPROnly(AMDGPU::VGPRRegBankID, LoadTy); + ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); PtrMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize); } @@ -2449,11 +3353,35 @@ AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg, /// in RegBankSelect::Mode::Fast. Any mapping that would cause a /// VGPR to SGPR generated is illegal. /// +// Operands that must be SGPRs must accept potentially divergent VGPRs as +// legal. These will be dealt with in applyMappingImpl. +// const RegisterBankInfo::InstructionMapping & AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { const MachineFunction &MF = *MI.getParent()->getParent(); const MachineRegisterInfo &MRI = MF.getRegInfo(); + if (MI.isCopy()) { + // The default logic bothers to analyze impossible alternative mappings. We + // want the most straightforward mapping, so just directly handle this. + const RegisterBank *DstBank = getRegBank(MI.getOperand(0).getReg(), MRI, + *TRI); + const RegisterBank *SrcBank = getRegBank(MI.getOperand(1).getReg(), MRI, + *TRI); + assert(SrcBank && "src bank should have been assigned already"); + if (!DstBank) + DstBank = SrcBank; + + unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); + if (cannotCopy(*DstBank, *SrcBank, Size)) + return getInvalidInstructionMapping(); + + const ValueMapping &ValMap = getValueMapping(0, Size, *DstBank); + return getInstructionMapping( + 1, /*Cost*/ 1, + /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1); + } + if (MI.isRegSequence()) { // If any input is a VGPR, the result must be a VGPR. The default handling // assumes any copy between banks is legal. @@ -2592,6 +3520,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { LLVM_FALLTHROUGH; } case AMDGPU::G_PTR_ADD: + case AMDGPU::G_PTRMASK: case AMDGPU::G_ADD: case AMDGPU::G_SUB: case AMDGPU::G_MUL: @@ -2608,6 +3537,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_SMAX: case AMDGPU::G_UMIN: case AMDGPU::G_UMAX: + case AMDGPU::G_SHUFFLE_VECTOR: if (isSALUMapping(MI)) return getDefaultMappingSOP(MI); LLVM_FALLTHROUGH; @@ -2635,7 +3565,16 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_FMAXNUM_IEEE: case AMDGPU::G_FCANONICALIZE: case AMDGPU::G_INTRINSIC_TRUNC: + case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar? + case AMDGPU::G_FSHR: // TODO: Expand for scalar case AMDGPU::G_AMDGPU_FFBH_U32: + case AMDGPU::G_AMDGPU_FMIN_LEGACY: + case AMDGPU::G_AMDGPU_FMAX_LEGACY: + case AMDGPU::G_AMDGPU_RCP_IFLAG: + case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0: + case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1: + case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2: + case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3: return getDefaultMappingVOP(MI); case AMDGPU::G_UMULH: case AMDGPU::G_SMULH: { @@ -2664,6 +3603,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); break; } + case AMDGPU::G_DYN_STACKALLOC: { + // Result is always uniform, and a wave reduction is needed for the source. + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); + unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI); + OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32); + break; + } case AMDGPU::G_INSERT: { unsigned BankID = isSALUMapping(MI) ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; @@ -2719,12 +3665,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_BITCAST: case AMDGPU::G_INTTOPTR: case AMDGPU::G_PTRTOINT: - case AMDGPU::G_CTLZ: - case AMDGPU::G_CTLZ_ZERO_UNDEF: - case AMDGPU::G_CTTZ: - case AMDGPU::G_CTTZ_ZERO_UNDEF: - case AMDGPU::G_CTPOP: - case AMDGPU::G_BSWAP: case AMDGPU::G_BITREVERSE: case AMDGPU::G_FABS: case AMDGPU::G_FNEG: { @@ -2733,21 +3673,33 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); break; } + case AMDGPU::G_CTLZ_ZERO_UNDEF: + case AMDGPU::G_CTTZ_ZERO_UNDEF: + case AMDGPU::G_CTPOP: { + unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); + unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI); + OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32); + + // This should really be getValueMappingSGPR64Only, but allowing the generic + // code to handle the register split just makes using LegalizerHelper more + // difficult. + OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); + break; + } case AMDGPU::G_TRUNC: { Register Dst = MI.getOperand(0).getReg(); Register Src = MI.getOperand(1).getReg(); unsigned Bank = getRegBankID(Src, MRI, *TRI); unsigned DstSize = getSizeInBits(Dst, MRI, *TRI); unsigned SrcSize = getSizeInBits(Src, MRI, *TRI); - OpdsMapping[0] = DstSize == 1 && Bank != AMDGPU::SGPRRegBankID ? - AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize) : - AMDGPU::getValueMapping(Bank, DstSize); + OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize); break; } case AMDGPU::G_ZEXT: case AMDGPU::G_SEXT: - case AMDGPU::G_ANYEXT: { + case AMDGPU::G_ANYEXT: + case AMDGPU::G_SEXT_INREG: { Register Dst = MI.getOperand(0).getReg(); Register Src = MI.getOperand(1).getReg(); unsigned DstSize = getSizeInBits(Dst, MRI, *TRI); @@ -2765,17 +3717,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { break; } - // TODO: Should anyext be split into 32-bit part as well? - if (MI.getOpcode() == AMDGPU::G_ANYEXT) { - OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, DstSize); - OpdsMapping[1] = AMDGPU::getValueMapping(SrcBank->getID(), SrcSize); - } else { - // Scalar extend can use 64-bit BFE, but VGPRs require extending to - // 32-bits, and then to 64. - OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize); - OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(), - SrcSize); - } + // Scalar extend can use 64-bit BFE, but VGPRs require extending to + // 32-bits, and then to 64. + OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize); + OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(), + SrcSize); break; } case AMDGPU::G_FCMP: { @@ -2790,43 +3736,43 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_STORE: { assert(MI.getOperand(0).isReg()); unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); - // FIXME: We need to specify a different reg bank once scalar stores - // are supported. + + // FIXME: We need to specify a different reg bank once scalar stores are + // supported. const ValueMapping *ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); - // FIXME: Depending on the type of store, the pointer could be in - // the SGPR Reg bank. - // FIXME: Pointer size should be based on the address space. - const ValueMapping *PtrMapping = - AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 64); - OpdsMapping[0] = ValMapping; - OpdsMapping[1] = PtrMapping; + OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); break; } - case AMDGPU::G_ICMP: { auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); + + // See if the result register has already been constrained to vcc, which may + // happen due to control flow intrinsic lowering. + unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI, + AMDGPU::SGPRRegBankID); unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI); unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI); - bool CanUseSCC = Op2Bank == AMDGPU::SGPRRegBankID && + bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID && + Op2Bank == AMDGPU::SGPRRegBankID && Op3Bank == AMDGPU::SGPRRegBankID && (Size == 32 || (Size == 64 && (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) && Subtarget.hasScalarCompareEq64())); - unsigned Op0Bank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; + DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; + unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; // TODO: Use 32-bit for scalar output size. // SCC results will need to be copied to a 32-bit SGPR virtual register. const unsigned ResultSize = 1; - OpdsMapping[0] = AMDGPU::getValueMapping(Op0Bank, ResultSize); - OpdsMapping[1] = nullptr; // Predicate Operand. - OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size); - OpdsMapping[3] = AMDGPU::getValueMapping(Op3Bank, Size); + OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize); + OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size); + OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank, Size); break; } case AMDGPU::G_EXTRACT_VECTOR_ELT: { @@ -2852,15 +3798,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { unsigned VecSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); unsigned IdxSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits(); - unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI); unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI); unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI); OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize); - OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, VecSize); - OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(InsertEltBankID, - InsertSize); + OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize); + + // This is a weird case, because we need to break down the mapping based on + // the register bank of a different operand. + if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) { + OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID, + InsertSize); + } else { + assert(InsertSize == 32 || InsertSize == 64); + OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize); + } // The index can be either if the source vector is VGPR. OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize); @@ -2878,6 +3831,116 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { } break; } + case AMDGPU::G_AMDGPU_BUFFER_LOAD: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: + case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: + case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: + case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: + case AMDGPU::G_AMDGPU_BUFFER_STORE: + case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: + case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: + case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: + case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: { + OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + + // rsrc + OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); + + // vindex + OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + + // voffset + OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + + // soffset + OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); + + // Any remaining operands are immediates and were correctly null + // initialized. + break; + } + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: { + // vdata_out + OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + + // vdata_in + OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); + + // rsrc + OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + + // vindex + OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + + // voffset + OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); + + // soffset + OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); + + // Any remaining operands are immediates and were correctly null + // initialized. + break; + } + case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { + // vdata_out + OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + + // vdata_in + OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); + + // cmp + OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + + // rsrc + OpdsMapping[3] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + + // vindex + OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); + + // voffset + OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); + + // soffset + OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI); + + // Any remaining operands are immediates and were correctly null + // initialized. + break; + } + case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: { + // Lie and claim everything is legal, even though some need to be + // SGPRs. applyMapping will have to deal with it as a waterfall loop. + OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); + OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + + // We need to convert this to a MUBUF if either the resource of offset is + // VGPR. + unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID(); + unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID(); + unsigned ResultBank = regBankUnion(RSrcBank, OffsetBank); + + unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0); + break; + } case AMDGPU::G_INTRINSIC: { switch (MI.getIntrinsicID()) { default: @@ -2890,9 +3953,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_log_clamp: case Intrinsic::amdgcn_rcp: case Intrinsic::amdgcn_rcp_legacy: + case Intrinsic::amdgcn_sqrt: case Intrinsic::amdgcn_rsq: case Intrinsic::amdgcn_rsq_legacy: case Intrinsic::amdgcn_rsq_clamp: + case Intrinsic::amdgcn_fmul_legacy: case Intrinsic::amdgcn_ldexp: case Intrinsic::amdgcn_frexp_mant: case Intrinsic::amdgcn_frexp_exp: @@ -2911,8 +3976,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_fmad_ftz: case Intrinsic::amdgcn_mbcnt_lo: case Intrinsic::amdgcn_mbcnt_hi: - case Intrinsic::amdgcn_ubfe: - case Intrinsic::amdgcn_sbfe: case Intrinsic::amdgcn_mul_u24: case Intrinsic::amdgcn_mul_i24: case Intrinsic::amdgcn_lerp: @@ -2933,13 +3996,21 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_udot4: case Intrinsic::amdgcn_sdot8: case Intrinsic::amdgcn_udot8: - case Intrinsic::amdgcn_wwm: - case Intrinsic::amdgcn_wqm: + return getDefaultMappingVOP(MI); + case Intrinsic::amdgcn_sbfe: + case Intrinsic::amdgcn_ubfe: + if (isSALUMapping(MI)) + return getDefaultMappingSOP(MI); return getDefaultMappingVOP(MI); case Intrinsic::amdgcn_ds_swizzle: case Intrinsic::amdgcn_ds_permute: case Intrinsic::amdgcn_ds_bpermute: case Intrinsic::amdgcn_update_dpp: + case Intrinsic::amdgcn_mov_dpp8: + case Intrinsic::amdgcn_mov_dpp: + case Intrinsic::amdgcn_wwm: + case Intrinsic::amdgcn_wqm: + case Intrinsic::amdgcn_softwqm: return getDefaultMappingAllVGPR(MI); case Intrinsic::amdgcn_kernarg_segment_ptr: case Intrinsic::amdgcn_s_getpc: @@ -2954,26 +4025,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size); break; } - case Intrinsic::amdgcn_s_buffer_load: { - // FIXME: This should be moved to G_INTRINSIC_W_SIDE_EFFECTS - Register RSrc = MI.getOperand(2).getReg(); // SGPR - Register Offset = MI.getOperand(3).getReg(); // SGPR/imm - - unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); - unsigned Size2 = MRI.getType(RSrc).getSizeInBits(); - unsigned Size3 = MRI.getType(Offset).getSizeInBits(); - - unsigned RSrcBank = getRegBankID(RSrc, MRI, *TRI); - unsigned OffsetBank = getRegBankID(Offset, MRI, *TRI); - - OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size0); - OpdsMapping[1] = nullptr; // intrinsic id - - // Lie and claim everything is legal, even though some need to be - // SGPRs. applyMapping will have to deal with it as a waterfall loop. - OpdsMapping[2] = AMDGPU::getValueMapping(RSrcBank, Size2); // rsrc - OpdsMapping[3] = AMDGPU::getValueMapping(OffsetBank, Size3); - OpdsMapping[4] = nullptr; + case Intrinsic::amdgcn_ps_live: { + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); break; } case Intrinsic::amdgcn_div_scale: { @@ -2983,11 +4036,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size); unsigned SrcSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits(); - OpdsMapping[3] = AMDGPU::getValueMapping( - getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI), SrcSize); - OpdsMapping[4] = AMDGPU::getValueMapping( - getRegBankID(MI.getOperand(4).getReg(), MRI, *TRI), SrcSize); - + OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); + OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); break; } case Intrinsic::amdgcn_class: { @@ -2997,10 +4047,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { unsigned Src1Size = MRI.getType(Src1Reg).getSizeInBits(); unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize); - OpdsMapping[2] = AMDGPU::getValueMapping(getRegBankID(Src0Reg, MRI, *TRI), - Src0Size); - OpdsMapping[3] = AMDGPU::getValueMapping(getRegBankID(Src1Reg, MRI, *TRI), - Src1Size); + OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size); + OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size); break; } case Intrinsic::amdgcn_icmp: @@ -3009,10 +4057,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { // This is not VCCRegBank because this is not used in boolean contexts. OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); - unsigned Op1Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI); - unsigned Op2Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI); - OpdsMapping[2] = AMDGPU::getValueMapping(Op1Bank, OpSize); - OpdsMapping[3] = AMDGPU::getValueMapping(Op2Bank, OpSize); + OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); + OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); break; } case Intrinsic::amdgcn_readlane: { @@ -3054,6 +4100,16 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; } + case Intrinsic::amdgcn_permlane16: + case Intrinsic::amdgcn_permlanex16: { + unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); + OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); + OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); + OpdsMapping[4] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + OpdsMapping[5] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); + break; + } case Intrinsic::amdgcn_mfma_f32_4x4x1f32: case Intrinsic::amdgcn_mfma_f32_4x4x4f16: case Intrinsic::amdgcn_mfma_i32_4x4x4i8: @@ -3086,9 +4142,46 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); break; } + case Intrinsic::amdgcn_interp_p1: + case Intrinsic::amdgcn_interp_p2: + case Intrinsic::amdgcn_interp_mov: + case Intrinsic::amdgcn_interp_p1_f16: + case Intrinsic::amdgcn_interp_p2_f16: { + const int M0Idx = MI.getNumOperands() - 1; + Register M0Reg = MI.getOperand(M0Idx).getReg(); + unsigned M0Bank = getRegBankID(M0Reg, MRI, *TRI, AMDGPU::SGPRRegBankID); + unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); + for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I) + OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); + + // Must be SGPR, but we must take whatever the original bank is and fix it + // later. + OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32); + break; + } + case Intrinsic::amdgcn_ballot: { + unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); + OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize); + break; + } } break; } + case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: + case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: { + auto IntrID = MI.getIntrinsicID(); + const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(IntrID); + assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic"); + // Non-images can have complications from operands that allow both SGPR + // and VGPR. For now it's too complicated to figure out the final opcode + // to derive the register bank from the MCInstrDesc. + assert(RSrcIntrin->IsImage); + return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg); + } case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: { auto IntrID = MI.getIntrinsicID(); switch (IntrID) { @@ -3100,13 +4193,9 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; } - case Intrinsic::amdgcn_ds_append: - case Intrinsic::amdgcn_ds_consume: case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmax: - case Intrinsic::amdgcn_atomic_inc: - case Intrinsic::amdgcn_atomic_dec: return getDefaultMappingAllVGPR(MI); case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_swap: { @@ -3118,17 +4207,16 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); break; } + case Intrinsic::amdgcn_ds_append: + case Intrinsic::amdgcn_ds_consume: { + unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); + OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + break; + } case Intrinsic::amdgcn_exp_compr: - OpdsMapping[0] = nullptr; // IntrinsicID - // FIXME: These are immediate values which can't be read from registers. - OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); - OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); - // FIXME: Could we support packed types here? OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); - // FIXME: These are immediate values which can't be read from registers. - OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); - OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); break; case Intrinsic::amdgcn_exp: // FIXME: Could we support packed types here? @@ -3137,31 +4225,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); break; - case Intrinsic::amdgcn_buffer_load: { - Register RSrc = MI.getOperand(2).getReg(); // SGPR - Register VIndex = MI.getOperand(3).getReg(); // VGPR - Register Offset = MI.getOperand(4).getReg(); // SGPR/VGPR/imm - - unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); - unsigned Size2 = MRI.getType(RSrc).getSizeInBits(); - unsigned Size3 = MRI.getType(VIndex).getSizeInBits(); - unsigned Size4 = MRI.getType(Offset).getSizeInBits(); - - unsigned RSrcBank = getRegBankID(RSrc, MRI, *TRI); - unsigned OffsetBank = getRegBankID(Offset, MRI, *TRI); - - OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size0); - OpdsMapping[1] = nullptr; // intrinsic id - - // Lie and claim everything is legal, even though some need to be - // SGPRs. applyMapping will have to deal with it as a waterfall loop. - OpdsMapping[2] = AMDGPU::getValueMapping(RSrcBank, Size2); // rsrc - OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size3); - OpdsMapping[4] = AMDGPU::getValueMapping(OffsetBank, Size4); - OpdsMapping[5] = nullptr; - OpdsMapping[6] = nullptr; - break; - } case Intrinsic::amdgcn_s_sendmsg: case Intrinsic::amdgcn_s_sendmsghalt: { // This must be an SGPR, but accept a VGPR. @@ -3170,8 +4233,14 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); break; } - case Intrinsic::amdgcn_end_cf: - case Intrinsic::amdgcn_init_exec: { + case Intrinsic::amdgcn_s_setreg: { + // This must be an SGPR, but accept a VGPR. + unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI, + AMDGPU::SGPRRegBankID); + OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); + break; + } + case Intrinsic::amdgcn_end_cf: { unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; @@ -3227,7 +4296,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_init_exec_from_input: { unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); - OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; } case Intrinsic::amdgcn_ds_gws_init: @@ -3251,15 +4319,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { break; } default: - if (const AMDGPU::RsrcIntrinsic *RSrcIntrin = - AMDGPU::lookupRsrcIntrinsic(IntrID)) { - // Non-images can have complications from operands that allow both SGPR - // and VGPR. For now it's too complicated to figure out the final opcode - // to derive the register bank from the MCInstrDesc. - if (RSrcIntrin->IsImage) - return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg); - } - return getInvalidInstructionMapping(); } break; @@ -3319,9 +4378,20 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_ATOMICRMW_UMAX: case AMDGPU::G_ATOMICRMW_UMIN: case AMDGPU::G_ATOMICRMW_FADD: - case AMDGPU::G_ATOMIC_CMPXCHG: - case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: { - return getDefaultMappingAllVGPR(MI); + case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: + case AMDGPU::G_AMDGPU_ATOMIC_INC: + case AMDGPU::G_AMDGPU_ATOMIC_DEC: { + OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); + OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + break; + } + case AMDGPU::G_ATOMIC_CMPXCHG: { + OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); + OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + break; } case AMDGPU::G_BRCOND: { unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI, |