diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 684 |
1 files changed, 502 insertions, 182 deletions
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 683be871ff82..e4ab72f1095b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -28,10 +28,15 @@ #define GET_INSTRMAP_INFO #include "AMDGPUGenInstrInfo.inc" -static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( - "amdhsa-code-object-version", llvm::cl::Hidden, - llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), - llvm::cl::ZeroOrMore); +static llvm::cl::opt<unsigned> + AmdhsaCodeObjectVersion("amdhsa-code-object-version", llvm::cl::Hidden, + llvm::cl::desc("AMDHSA Code Object Version"), + llvm::cl::init(4)); + +// TODO-GFX11: Remove this when full 16-bit codegen is implemented. +static llvm::cl::opt<bool> + LimitTo128VGPRs("amdgpu-limit-to-128-vgprs", llvm::cl::Hidden, + llvm::cl::desc("Never use more than 128 VGPRs")); namespace { @@ -44,9 +49,8 @@ unsigned getBitMask(unsigned Shift, unsigned Width) { /// /// \returns Packed \p Dst. unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { - Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); - Dst |= (Src << Shift) & getBitMask(Shift, Width); - return Dst; + unsigned Mask = getBitMask(Shift, Width); + return ((Src << Shift) & Mask) | (Dst & ~Mask); } /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. @@ -57,30 +61,40 @@ unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { } /// \returns Vmcnt bit shift (lower bits). -unsigned getVmcntBitShiftLo() { return 0; } +unsigned getVmcntBitShiftLo(unsigned VersionMajor) { + return VersionMajor >= 11 ? 10 : 0; +} /// \returns Vmcnt bit width (lower bits). -unsigned getVmcntBitWidthLo() { return 4; } +unsigned getVmcntBitWidthLo(unsigned VersionMajor) { + return VersionMajor >= 11 ? 6 : 4; +} /// \returns Expcnt bit shift. -unsigned getExpcntBitShift() { return 4; } +unsigned getExpcntBitShift(unsigned VersionMajor) { + return VersionMajor >= 11 ? 0 : 4; +} /// \returns Expcnt bit width. -unsigned getExpcntBitWidth() { return 3; } +unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; } /// \returns Lgkmcnt bit shift. -unsigned getLgkmcntBitShift() { return 8; } +unsigned getLgkmcntBitShift(unsigned VersionMajor) { + return VersionMajor >= 11 ? 4 : 8; +} /// \returns Lgkmcnt bit width. unsigned getLgkmcntBitWidth(unsigned VersionMajor) { - return (VersionMajor >= 10) ? 6 : 4; + return VersionMajor >= 10 ? 6 : 4; } /// \returns Vmcnt bit shift (higher bits). -unsigned getVmcntBitShiftHi() { return 14; } +unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; } /// \returns Vmcnt bit width (higher bits). -unsigned getVmcntBitWidthHi() { return 2; } +unsigned getVmcntBitWidthHi(unsigned VersionMajor) { + return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0; +} } // end namespace anonymous @@ -136,6 +150,41 @@ bool isHsaAbiVersion3AndAbove(const MCSubtargetInfo *STI) { isHsaAbiVersion5(STI); } +unsigned getAmdhsaCodeObjectVersion() { + return AmdhsaCodeObjectVersion; +} + +unsigned getMultigridSyncArgImplicitArgPosition() { + switch (AmdhsaCodeObjectVersion) { + case 2: + case 3: + case 4: + return 48; + case 5: + return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET; + default: + llvm_unreachable("Unexpected code object version"); + return 0; + } +} + + +// FIXME: All such magic numbers about the ABI should be in a +// central TD file. +unsigned getHostcallImplicitArgPosition() { + switch (AmdhsaCodeObjectVersion) { + case 2: + case 3: + case 4: + return 24; + case 5: + return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET; + default: + llvm_unreachable("Unexpected code object version"); + return 0; + } +} + #define GET_MIMGBaseOpcodesTable_IMPL #define GET_MIMGDimInfoTable_IMPL #define GET_MIMGInfoTable_IMPL @@ -144,6 +193,7 @@ bool isHsaAbiVersion3AndAbove(const MCSubtargetInfo *STI) { #define GET_MIMGBiasMappingTable_IMPL #define GET_MIMGOffsetMappingTable_IMPL #define GET_MIMGG16MappingTable_IMPL +#define GET_MAIInstInfoTable_IMPL #include "AMDGPUGenSearchableTables.inc" int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, @@ -223,6 +273,10 @@ struct VOPInfo { bool IsSingle; }; +struct VOPC64DPPInfo { + uint16_t Opcode; +}; + #define GET_MTBUFInfoTable_DECL #define GET_MTBUFInfoTable_IMPL #define GET_MUBUFInfoTable_DECL @@ -235,6 +289,14 @@ struct VOPInfo { #define GET_VOP2InfoTable_IMPL #define GET_VOP3InfoTable_DECL #define GET_VOP3InfoTable_IMPL +#define GET_VOPC64DPPTable_DECL +#define GET_VOPC64DPPTable_IMPL +#define GET_VOPC64DPP8Table_DECL +#define GET_VOPC64DPP8Table_IMPL +#define GET_WMMAOpcode2AddrMappingTable_DECL +#define GET_WMMAOpcode2AddrMappingTable_IMPL +#define GET_WMMAOpcode3AddrMappingTable_DECL +#define GET_WMMAOpcode3AddrMappingTable_IMPL #include "AMDGPUGenSearchableTables.inc" int getMTBUFBaseOpcode(unsigned Opc) { @@ -322,6 +384,30 @@ bool getVOP3IsSingle(unsigned Opc) { return Info ? Info->IsSingle : false; } +bool isVOPC64DPP(unsigned Opc) { + return isVOPC64DPPOpcodeHelper(Opc) || isVOPC64DPP8OpcodeHelper(Opc); +} + +bool getMAIIsDGEMM(unsigned Opc) { + const MAIInstInfo *Info = getMAIInstInfoHelper(Opc); + return Info ? Info->is_dgemm : false; +} + +bool getMAIIsGFX940XDL(unsigned Opc) { + const MAIInstInfo *Info = getMAIInstInfoHelper(Opc); + return Info ? Info->is_gfx940_xdl : false; +} + +unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) { + const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opc); + return Info ? Info->Opcode3Addr : ~0u; +} + +unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc) { + const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom3AddrOpcode(Opc); + return Info ? Info->Opcode2Addr : ~0u; +} + // Wrapper for Tablegen'd function. enum Subtarget is not defined in any // header files, so we need to wrap it in a function that takes unsigned // instead. @@ -740,6 +826,15 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { } unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { + if (LimitTo128VGPRs.getNumOccurrences() ? LimitTo128VGPRs + : isGFX11Plus(*STI)) { + // GFX11 changes the encoding of 16-bit operands in VOP1/2/C instructions + // such that values 128..255 no longer mean v128..v255, they mean + // v0.hi..v127.hi instead. Until the compiler understands this, it is not + // safe to use v128..v255. + // TODO-GFX11: Remove this when full 16-bit codegen is implemented. + return 128; + } if (STI->getFeatureBits().test(FeatureGFX90AInsts)) return 512; return 256; @@ -904,16 +999,13 @@ std::pair<int, int> getIntegerPairAttribute(const Function &F, } unsigned getVmcntBitMask(const IsaVersion &Version) { - unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; - if (Version.Major < 9) - return VmcntLo; - - unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); - return VmcntLo | VmcntHi; + return (1 << (getVmcntBitWidthLo(Version.Major) + + getVmcntBitWidthHi(Version.Major))) - + 1; } unsigned getExpcntBitMask(const IsaVersion &Version) { - return (1 << getExpcntBitWidth()) - 1; + return (1 << getExpcntBitWidth(Version.Major)) - 1; } unsigned getLgkmcntBitMask(const IsaVersion &Version) { @@ -921,36 +1013,32 @@ unsigned getLgkmcntBitMask(const IsaVersion &Version) { } unsigned getWaitcntBitMask(const IsaVersion &Version) { - unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); - unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); - unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), + unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(Version.Major), + getVmcntBitWidthLo(Version.Major)); + unsigned Expcnt = getBitMask(getExpcntBitShift(Version.Major), + getExpcntBitWidth(Version.Major)); + unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(Version.Major), getLgkmcntBitWidth(Version.Major)); - unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; - if (Version.Major < 9) - return Waitcnt; - - unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); - return Waitcnt | VmcntHi; + unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(Version.Major), + getVmcntBitWidthHi(Version.Major)); + return VmcntLo | Expcnt | Lgkmcnt | VmcntHi; } unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { - unsigned VmcntLo = - unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); - if (Version.Major < 9) - return VmcntLo; - - unsigned VmcntHi = - unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); - VmcntHi <<= getVmcntBitWidthLo(); - return VmcntLo | VmcntHi; + unsigned VmcntLo = unpackBits(Waitcnt, getVmcntBitShiftLo(Version.Major), + getVmcntBitWidthLo(Version.Major)); + unsigned VmcntHi = unpackBits(Waitcnt, getVmcntBitShiftHi(Version.Major), + getVmcntBitWidthHi(Version.Major)); + return VmcntLo | VmcntHi << getVmcntBitWidthLo(Version.Major); } unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { - return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); + return unpackBits(Waitcnt, getExpcntBitShift(Version.Major), + getExpcntBitWidth(Version.Major)); } unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { - return unpackBits(Waitcnt, getLgkmcntBitShift(), + return unpackBits(Waitcnt, getLgkmcntBitShift(Version.Major), getLgkmcntBitWidth(Version.Major)); } @@ -971,24 +1059,23 @@ Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt) { - Waitcnt = - packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); - if (Version.Major < 9) - return Waitcnt; - - Vmcnt >>= getVmcntBitWidthLo(); - return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); + Waitcnt = packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(Version.Major), + getVmcntBitWidthLo(Version.Major)); + return packBits(Vmcnt >> getVmcntBitWidthLo(Version.Major), Waitcnt, + getVmcntBitShiftHi(Version.Major), + getVmcntBitWidthHi(Version.Major)); } unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt) { - return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); + return packBits(Expcnt, Waitcnt, getExpcntBitShift(Version.Major), + getExpcntBitWidth(Version.Major)); } unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt) { - return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), - getLgkmcntBitWidth(Version.Major)); + return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(Version.Major), + getLgkmcntBitWidth(Version.Major)); } unsigned encodeWaitcnt(const IsaVersion &Version, @@ -1005,43 +1092,184 @@ unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { } //===----------------------------------------------------------------------===// -// hwreg +// Custom Operands. +// +// A table of custom operands shall describe "primary" operand names +// first followed by aliases if any. It is not required but recommended +// to arrange operands so that operand encoding match operand position +// in the table. This will make disassembly a bit more efficient. +// Unused slots in the table shall have an empty name. +// //===----------------------------------------------------------------------===// -namespace Hwreg { +template <class T> +static bool isValidOpr(int Idx, const CustomOperand<T> OpInfo[], int OpInfoSize, + T Context) { + return 0 <= Idx && Idx < OpInfoSize && !OpInfo[Idx].Name.empty() && + (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context)); +} -int64_t getHwregId(const StringRef Name) { - for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { - if (IdSymbolic[Id] && Name == IdSymbolic[Id]) - return Id; +template <class T> +static int getOprIdx(std::function<bool(const CustomOperand<T> &)> Test, + const CustomOperand<T> OpInfo[], int OpInfoSize, + T Context) { + int InvalidIdx = OPR_ID_UNKNOWN; + for (int Idx = 0; Idx < OpInfoSize; ++Idx) { + if (Test(OpInfo[Idx])) { + if (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context)) + return Idx; + InvalidIdx = OPR_ID_UNSUPPORTED; + } } - return ID_UNKNOWN_; + return InvalidIdx; } -static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { - if (isSI(STI) || isCI(STI) || isVI(STI)) - return ID_SYMBOLIC_FIRST_GFX9_; - else if (isGFX9(STI)) - return ID_SYMBOLIC_FIRST_GFX10_; - else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) - return ID_SYMBOLIC_FIRST_GFX1030_; - else - return ID_SYMBOLIC_LAST_; +template <class T> +static int getOprIdx(const StringRef Name, const CustomOperand<T> OpInfo[], + int OpInfoSize, T Context) { + auto Test = [=](const CustomOperand<T> &Op) { return Op.Name == Name; }; + return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context); } -bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { - switch (Id) { - case ID_HW_ID: - return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); - case ID_HW_ID1: - case ID_HW_ID2: - return isGFX10Plus(STI); - case ID_XNACK_MASK: - return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); - default: - return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && - IdSymbolic[Id]; +template <class T> +static int getOprIdx(int Id, const CustomOperand<T> OpInfo[], int OpInfoSize, + T Context, bool QuickCheck = true) { + auto Test = [=](const CustomOperand<T> &Op) { + return Op.Encoding == Id && !Op.Name.empty(); + }; + // This is an optimization that should work in most cases. + // As a side effect, it may cause selection of an alias + // instead of a primary operand name in case of sparse tables. + if (QuickCheck && isValidOpr<T>(Id, OpInfo, OpInfoSize, Context) && + OpInfo[Id].Encoding == Id) { + return Id; + } + return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context); +} + +//===----------------------------------------------------------------------===// +// Custom Operand Values +//===----------------------------------------------------------------------===// + +static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr, + int Size, + const MCSubtargetInfo &STI) { + unsigned Enc = 0; + for (int Idx = 0; Idx < Size; ++Idx) { + const auto &Op = Opr[Idx]; + if (Op.isSupported(STI)) + Enc |= Op.encode(Op.Default); + } + return Enc; +} + +static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr, + int Size, unsigned Code, + bool &HasNonDefaultVal, + const MCSubtargetInfo &STI) { + unsigned UsedOprMask = 0; + HasNonDefaultVal = false; + for (int Idx = 0; Idx < Size; ++Idx) { + const auto &Op = Opr[Idx]; + if (!Op.isSupported(STI)) + continue; + UsedOprMask |= Op.getMask(); + unsigned Val = Op.decode(Code); + if (!Op.isValid(Val)) + return false; + HasNonDefaultVal |= (Val != Op.Default); } + return (Code & ~UsedOprMask) == 0; +} + +static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size, + unsigned Code, int &Idx, StringRef &Name, + unsigned &Val, bool &IsDefault, + const MCSubtargetInfo &STI) { + while (Idx < Size) { + const auto &Op = Opr[Idx++]; + if (Op.isSupported(STI)) { + Name = Op.Name; + Val = Op.decode(Code); + IsDefault = (Val == Op.Default); + return true; + } + } + + return false; +} + +static int encodeCustomOperandVal(const CustomOperandVal &Op, + int64_t InputVal) { + if (InputVal < 0 || InputVal > Op.Max) + return OPR_VAL_INVALID; + return Op.encode(InputVal); +} + +static int encodeCustomOperand(const CustomOperandVal *Opr, int Size, + const StringRef Name, int64_t InputVal, + unsigned &UsedOprMask, + const MCSubtargetInfo &STI) { + int InvalidId = OPR_ID_UNKNOWN; + for (int Idx = 0; Idx < Size; ++Idx) { + const auto &Op = Opr[Idx]; + if (Op.Name == Name) { + if (!Op.isSupported(STI)) { + InvalidId = OPR_ID_UNSUPPORTED; + continue; + } + auto OprMask = Op.getMask(); + if (OprMask & UsedOprMask) + return OPR_ID_DUPLICATE; + UsedOprMask |= OprMask; + return encodeCustomOperandVal(Op, InputVal); + } + } + return InvalidId; +} + +//===----------------------------------------------------------------------===// +// DepCtr +//===----------------------------------------------------------------------===// + +namespace DepCtr { + +int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) { + static int Default = -1; + if (Default == -1) + Default = getDefaultCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, STI); + return Default; +} + +bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal, + const MCSubtargetInfo &STI) { + return isSymbolicCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, Code, + HasNonDefaultVal, STI); +} + +bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val, + bool &IsDefault, const MCSubtargetInfo &STI) { + return decodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Code, Id, Name, Val, + IsDefault, STI); +} + +int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, + const MCSubtargetInfo &STI) { + return encodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Name, Val, UsedOprMask, + STI); +} + +} // namespace DepCtr + +//===----------------------------------------------------------------------===// +// hwreg +//===----------------------------------------------------------------------===// + +namespace Hwreg { + +int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI) { + int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Opr, OPR_SIZE, STI); + return (Idx < 0) ? Idx : Opr[Idx].Encoding; } bool isValidHwreg(int64_t Id) { @@ -1063,7 +1291,8 @@ uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { } StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { - return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; + int Idx = getOprIdx<const MCSubtargetInfo &>(Id, Opr, OPR_SIZE, STI); + return (Idx < 0) ? "" : Opr[Idx].Name; } void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { @@ -1087,12 +1316,13 @@ struct ExpTgt { }; static constexpr ExpTgt ExpTgtInfo[] = { - {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, - {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, - {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, - {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, - {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, - {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, + {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, + {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, + {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, + {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, + {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, + {{"dual_src_blend"}, ET_DUAL_SRC_BLEND0, ET_DUAL_SRC_BLEND_MAX_IDX}, + {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, }; bool getTgtName(unsigned Id, StringRef &Name, int &Index) { @@ -1130,7 +1360,20 @@ unsigned getTgtId(const StringRef Name) { } bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { - return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); + switch (Id) { + case ET_NULL: + return !isGFX11Plus(STI); + case ET_POS4: + case ET_PRIM: + return isGFX10Plus(STI); + case ET_DUAL_SRC_BLEND0: + case ET_DUAL_SRC_BLEND1: + return isGFX11Plus(STI); + default: + if (Id >= ET_PARAM0 && Id <= ET_PARAM31) + return !isGFX11Plus(STI); + return true; + } } } // namespace Exp @@ -1196,27 +1439,44 @@ void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; } -int64_t getUnifiedFormat(const StringRef Name) { - for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { - if (Name == UfmtSymbolic[Id]) - return Id; +int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) { + if (isGFX11Plus(STI)) { + for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) { + if (Name == UfmtSymbolicGFX11[Id]) + return Id; + } + } else { + for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) { + if (Name == UfmtSymbolicGFX10[Id]) + return Id; + } } return UFMT_UNDEF; } -StringRef getUnifiedFormatName(unsigned Id) { - return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; +StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) { + if(isValidUnifiedFormat(Id, STI)) + return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id]; + return ""; } -bool isValidUnifiedFormat(unsigned Id) { - return Id <= UFMT_LAST; +bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) { + return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST; } -int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { +int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt, + const MCSubtargetInfo &STI) { int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); - for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { - if (Fmt == DfmtNfmt2UFmt[Id]) - return Id; + if (isGFX11Plus(STI)) { + for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) { + if (Fmt == DfmtNfmt2UFmtGFX11[Id]) + return Id; + } + } else { + for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) { + if (Fmt == DfmtNfmt2UFmtGFX10[Id]) + return Id; + } } return UFMT_UNDEF; } @@ -1239,40 +1499,22 @@ unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { namespace SendMsg { -int64_t getMsgId(const StringRef Name) { - for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { - if (IdSymbolic[i] && Name == IdSymbolic[i]) - return i; - } - return ID_UNKNOWN_; +static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) { + return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_; } -bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { - if (Strict) { - switch (MsgId) { - case ID_SAVEWAVE: - return isVI(STI) || isGFX9Plus(STI); - case ID_STALL_WAVE_GEN: - case ID_HALT_WAVES: - case ID_ORDERED_PS_DONE: - case ID_GS_ALLOC_REQ: - case ID_GET_DOORBELL: - return isGFX9Plus(STI); - case ID_EARLY_PRIM_DEALLOC: - return isGFX9(STI); - case ID_GET_DDID: - return isGFX10Plus(STI); - default: - return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; - } - } else { - return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); - } +int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI) { + int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Msg, MSG_SIZE, STI); + return (Idx < 0) ? Idx : Msg[Idx].Encoding; +} + +bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) { + return (MsgId & ~(getMsgIdMask(STI))) == 0; } -StringRef getMsgName(int64_t MsgId) { - assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); - return IdSymbolic[MsgId]; +StringRef getMsgName(int64_t MsgId, const MCSubtargetInfo &STI) { + int Idx = getOprIdx<const MCSubtargetInfo &>(MsgId, Msg, MSG_SIZE, STI); + return (Idx < 0) ? "" : Msg[Idx].Name; } int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { @@ -1289,26 +1531,27 @@ int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict) { - assert(isValidMsgId(MsgId, STI, Strict)); + assert(isValidMsgId(MsgId, STI)); if (!Strict) return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); - switch(MsgId) - { - case ID_GS: - return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; - case ID_GS_DONE: - return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; - case ID_SYSMSG: + if (MsgId == ID_SYSMSG) return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; - default: - return OpId == OP_NONE_; + if (!isGFX11Plus(STI)) { + switch (MsgId) { + case ID_GS_PreGFX11: + return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; + case ID_GS_DONE_PreGFX11: + return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; + } } + return OpId == OP_NONE_; } -StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { - assert(msgRequiresOp(MsgId)); +StringRef getMsgOpName(int64_t MsgId, int64_t OpId, + const MCSubtargetInfo &STI) { + assert(msgRequiresOp(MsgId, STI)); return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; } @@ -1319,42 +1562,48 @@ bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, if (!Strict) return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); - switch(MsgId) - { - case ID_GS: - return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; - case ID_GS_DONE: - return (OpId == OP_GS_NOP)? - (StreamId == STREAM_ID_NONE_) : - (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); - default: - return StreamId == STREAM_ID_NONE_; + if (!isGFX11Plus(STI)) { + switch (MsgId) { + case ID_GS_PreGFX11: + return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; + case ID_GS_DONE_PreGFX11: + return (OpId == OP_GS_NOP) ? + (StreamId == STREAM_ID_NONE_) : + (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); + } } + return StreamId == STREAM_ID_NONE_; } -bool msgRequiresOp(int64_t MsgId) { - return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; +bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) { + return MsgId == ID_SYSMSG || + (!isGFX11Plus(STI) && + (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11)); } -bool msgSupportsStream(int64_t MsgId, int64_t OpId) { - return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; +bool msgSupportsStream(int64_t MsgId, int64_t OpId, + const MCSubtargetInfo &STI) { + return !isGFX11Plus(STI) && + (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) && + OpId != OP_GS_NOP; } -void decodeMsg(unsigned Val, - uint16_t &MsgId, - uint16_t &OpId, - uint16_t &StreamId) { - MsgId = Val & ID_MASK_; - OpId = (Val & OP_MASK_) >> OP_SHIFT_; - StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; +void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId, + uint16_t &StreamId, const MCSubtargetInfo &STI) { + MsgId = Val & getMsgIdMask(STI); + if (isGFX11Plus(STI)) { + OpId = 0; + StreamId = 0; + } else { + OpId = (Val & OP_MASK_) >> OP_SHIFT_; + StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; + } } uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) { - return (MsgId << ID_SHIFT_) | - (OpId << OP_SHIFT_) | - (StreamId << STREAM_ID_SHIFT_); + return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_); } } // namespace SendMsg @@ -1427,6 +1676,10 @@ bool isModuleEntryFunctionCC(CallingConv::ID CC) { } } +bool isKernelCC(const Function *Func) { + return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv()); +} + bool hasXNACK(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; } @@ -1448,7 +1701,8 @@ bool hasG16(const MCSubtargetInfo &STI) { } bool hasPackedD16(const MCSubtargetInfo &STI) { - return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; + return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem] && !isCI(STI) && + !isSI(STI); } bool isSI(const MCSubtargetInfo &STI) { @@ -1467,6 +1721,18 @@ bool isGFX9(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; } +bool isGFX9_GFX10(const MCSubtargetInfo &STI) { + return isGFX9(STI) || isGFX10(STI); +} + +bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) { + return isVI(STI) || isGFX9(STI) || isGFX10(STI); +} + +bool isGFX8Plus(const MCSubtargetInfo &STI) { + return isVI(STI) || isGFX9Plus(STI); +} + bool isGFX9Plus(const MCSubtargetInfo &STI) { return isGFX9(STI) || isGFX10Plus(STI); } @@ -1475,7 +1741,29 @@ bool isGFX10(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; } -bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } +bool isGFX10Plus(const MCSubtargetInfo &STI) { + return isGFX10(STI) || isGFX11Plus(STI); +} + +bool isGFX11(const MCSubtargetInfo &STI) { + return STI.getFeatureBits()[AMDGPU::FeatureGFX11]; +} + +bool isGFX11Plus(const MCSubtargetInfo &STI) { + return isGFX11(STI); +} + +bool isNotGFX11Plus(const MCSubtargetInfo &STI) { + return !isGFX11Plus(STI); +} + +bool isNotGFX10Plus(const MCSubtargetInfo &STI) { + return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); +} + +bool isGFX10Before1030(const MCSubtargetInfo &STI) { + return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); +} bool isGCN3Encoding(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; @@ -1497,10 +1785,29 @@ bool isGFX90A(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; } +bool isGFX940(const MCSubtargetInfo &STI) { + return STI.getFeatureBits()[AMDGPU::FeatureGFX940Insts]; +} + bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; } +bool hasMAIInsts(const MCSubtargetInfo &STI) { + return STI.getFeatureBits()[AMDGPU::FeatureMAIInsts]; +} + +bool hasVOPD(const MCSubtargetInfo &STI) { + return STI.getFeatureBits()[AMDGPU::FeatureVOPD]; +} + +int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, + int32_t ArgNumVGPR) { + if (has90AInsts && ArgNumAGPR) + return alignTo(ArgNumVGPR, 4) + ArgNumAGPR; + return std::max(ArgNumVGPR, ArgNumAGPR); +} + bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); @@ -1508,13 +1815,6 @@ bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { Reg == AMDGPU::SCC; } -bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { - for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { - if (*R == Reg1) return true; - } - return false; -} - #define MAP_REG2REG \ using namespace AMDGPU; \ switch(Reg) { \ @@ -1554,6 +1854,9 @@ bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ + CASE_GFXPRE11_GFX11PLUS(M0) \ + CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \ + CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \ } #define CASE_CI_VI(node) \ @@ -1563,6 +1866,12 @@ bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { #define CASE_VI_GFX9PLUS(node) \ case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; +#define CASE_GFXPRE11_GFX11PLUS(node) \ + case node: return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11; + +#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \ + case node: return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11; + unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { if (STI.getTargetTriple().getArch() == Triple::r600) return Reg; @@ -1571,9 +1880,13 @@ unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { #undef CASE_CI_VI #undef CASE_VI_GFX9PLUS +#undef CASE_GFXPRE11_GFX11PLUS +#undef CASE_GFXPRE11_GFX11PLUS_TO #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; +#define CASE_GFXPRE11_GFX11PLUS(node) case node##_gfx11plus: case node##_gfxpre11: return node; +#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) unsigned mc2PseudoReg(unsigned Reg) { MAP_REG2REG @@ -1581,6 +1894,8 @@ unsigned mc2PseudoReg(unsigned Reg) { #undef CASE_CI_VI #undef CASE_VI_GFX9PLUS +#undef CASE_GFXPRE11_GFX11PLUS +#undef CASE_GFXPRE11_GFX11PLUS_TO #undef MAP_REG2REG bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { @@ -1934,7 +2249,7 @@ Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, } unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { - // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. + // Address offset is 12-bit signed for GFX10, 13-bit for GFX9 and GFX11+. if (AMDGPU::isGFX10(ST)) return Signed ? 12 : 11; @@ -2029,7 +2344,8 @@ const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); #define GET_SourcesOfDivergence_IMPL #define GET_Gfx9BufferFormat_IMPL -#define GET_Gfx10PlusBufferFormat_IMPL +#define GET_Gfx10BufferFormat_IMPL +#define GET_Gfx11PlusBufferFormat_IMPL #include "AMDGPUGenSearchableTables.inc" } // end anonymous namespace @@ -2042,16 +2358,20 @@ const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI) { - return isGFX10Plus(STI) - ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, + return isGFX11Plus(STI) + ? getGfx11PlusBufferFormatInfo(BitsPerComp, NumComponents, NumFormat) - : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); + : isGFX10(STI) ? getGfx10BufferFormatInfo(BitsPerComp, + NumComponents, NumFormat) + : getGfx9BufferFormatInfo(BitsPerComp, + NumComponents, NumFormat); } const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, const MCSubtargetInfo &STI) { - return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) - : getGfx9BufferFormatInfo(Format); + return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format) + : isGFX10(STI) ? getGfx10BufferFormatInfo(Format) + : getGfx9BufferFormatInfo(Format); } } // namespace AMDGPU |