diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SMInstructions.td')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SMInstructions.td | 104 |
1 files changed, 76 insertions, 28 deletions
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td index 79982d96c2c8e..70bf215c03f3f 100644 --- a/llvm/lib/Target/AMDGPU/SMInstructions.td +++ b/llvm/lib/Target/AMDGPU/SMInstructions.td @@ -1,4 +1,4 @@ -//===---- SMInstructions.td - Scalar Memory Instruction Defintions --------===// +//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,9 +11,11 @@ def smrd_offset_8 : NamedOperandU32<"SMRDOffset8", let OperandType = "OPERAND_IMMEDIATE"; } -def smrd_offset_20 : NamedOperandU32<"SMRDOffset20", - NamedMatchClass<"SMRDOffset20">> { +def smem_offset : NamedOperandU32<"SMEMOffset", + NamedMatchClass<"SMEMOffset">> { let OperandType = "OPERAND_IMMEDIATE"; + let EncoderMethod = "getSMEMOffsetEncoding"; + let DecoderMethod = "decodeSMEMOffset"; } //===----------------------------------------------------------------------===// @@ -43,6 +45,7 @@ class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> patt bit has_dlc = 0; bits<1> has_offset = 1; bits<1> offset_is_imm = 0; + bit is_buffer = 0; } class SM_Real <SM_Pseudo ps> @@ -51,9 +54,15 @@ class SM_Real <SM_Pseudo ps> let isPseudo = 0; let isCodeGenOnly = 0; + Instruction Opcode = !cast<Instruction>(NAME); + // copy relevant pseudo op flags let SubtargetPredicate = ps.SubtargetPredicate; let AsmMatchConverter = ps.AsmMatchConverter; + let UseNamedOperandTable = ps.UseNamedOperandTable; + let SMRD = ps.SMRD; + + bit is_buffer = ps.is_buffer; // encoding bits<7> sbase; @@ -153,7 +162,7 @@ multiclass SM_Pseudo_Stores<string opName, } multiclass SM_Pseudo_Discards<string opName> { - def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smrd_offset_20:$offset), 1>; + def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>; def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>; } @@ -178,14 +187,14 @@ class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pse class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< opName, (outs), (ins), "", [(node)]> { let hasSideEffects = 1; - let mayStore = 1; + let mayStore = 0; let has_sdst = 0; let has_sbase = 0; let has_offset = 0; } multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> { - def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smrd_offset_20:$offset), 1>; + def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>; def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>; } @@ -228,7 +237,7 @@ class SM_Pseudo_Atomic<string opName, SM_Atomic_Pseudo<opName, !if(isRet, (outs dataClass:$sdst), (outs)), !if(isImm, - (ins dataClass:$sdata, baseClass:$sbase, smrd_offset_20:$offset, DLC:$dlc), + (ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, DLC:$dlc), (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, DLC:$dlc)), !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", "") # "$dlc", isRet> { @@ -266,6 +275,7 @@ defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>; defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>; defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>; +let is_buffer = 1 in { defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads < "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC >; @@ -287,12 +297,14 @@ defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads < defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads < "s_buffer_load_dwordx16", SReg_128, SReg_512 >; +} let SubtargetPredicate = HasScalarStores in { defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>; defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>; defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>; +let is_buffer = 1 in { defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores < "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC >; @@ -304,8 +316,10 @@ defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores < defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores < "s_buffer_store_dwordx4", SReg_128, SReg_128 >; +} } // End SubtargetPredicate = HasScalarStores +let SubtargetPredicate = HasSMemTimeInst in def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; @@ -321,13 +335,16 @@ def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; defm S_ATC_PROBE : SM_Pseudo_Probe <"s_atc_probe", SReg_64>; +let is_buffer = 1 in { defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>; +} } // SubtargetPredicate = isGFX8Plus -let SubtargetPredicate = isGFX10Plus in { +let SubtargetPredicate = isGFX10Plus in def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; +let SubtargetPredicate = HasGetWaveIdInst in def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; -} // End SubtargetPredicate = isGFX10Plus + let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <"s_scratch_load_dword", SReg_64, SReg_32_XM0_XEXEC>; @@ -341,6 +358,7 @@ defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg let SubtargetPredicate = HasScalarAtomics in { +let is_buffer = 1 in { defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>; defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>; defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>; @@ -368,6 +386,7 @@ defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>; defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>; defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>; +} defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>; defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>; @@ -481,14 +500,17 @@ class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> let Inst{17} = imm; let Inst{25-18} = op; let Inst{31-26} = 0x30; //encoding - let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?); + + // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. + // Offset value is corrected accordingly when offset is encoded/decoded. + let Inst{52-32} = !if(ps.has_offset, offset{20-0}, ?); } multiclass SM_Real_Loads_vi<bits<8> op, string ps, SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { def _IMM_vi : SMEM_Real_vi <op, immPs> { - let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc); + let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); } def _SGPR_vi : SMEM_Real_vi <op, sgprPs> { let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); @@ -509,7 +531,7 @@ multiclass SM_Real_Stores_vi<bits<8> op, string ps, // FIXME: The operand name $offset is inconsistent with $soff used // in the pseudo def _IMM_vi : SMEM_Real_Store_vi <op, immPs> { - let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc); + let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); } def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> { @@ -665,12 +687,10 @@ class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc, DLC:$dlc); let LGKM_CNT = ps.LGKM_CNT; - let SMRD = ps.SMRD; let mayLoad = ps.mayLoad; let mayStore = ps.mayStore; let hasSideEffects = ps.hasSideEffects; let SchedRW = ps.SchedRW; - let UseNamedOperandTable = ps.UseNamedOperandTable; let Inst{7-0} = 0xff; let Inst{8} = 0; @@ -768,23 +788,26 @@ multiclass SMRD_Pattern <string Instr, ValueType vt> { multiclass SMLoad_Pattern <string Instr, ValueType vt> { // 1. Offset as an immediate def : GCNPat < - (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc, i1:$dlc), - (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, (as_i1imm $glc), - (as_i1imm $dlc))) - >; + (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), + (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_glc $cachepolicy), + (extract_dlc $cachepolicy)))> { + let AddedComplexity = 2; + } // 2. 32-bit IMM offset on CI def : GCNPat < - (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc, i1:$dlc)), - (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, (as_i1imm $glc), (as_i1imm $dlc))> { + (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), + (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, + (extract_glc $cachepolicy), (extract_dlc $cachepolicy))> { let OtherPredicates = [isGFX7Only]; + let AddedComplexity = 1; } // 3. Offset loaded in an 32bit SGPR def : GCNPat < - (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc, i1:$dlc), - (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, (as_i1imm $glc), - (as_i1imm $dlc))) + (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy), + (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_glc $cachepolicy), + (extract_dlc $cachepolicy))) >; } @@ -805,8 +828,13 @@ foreach vt = SReg_128.RegTypes in { defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; } -defm : SMRD_Pattern <"S_LOAD_DWORDX8", v8i32>; -defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>; +foreach vt = SReg_256.RegTypes in { +defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; +} + +foreach vt = SReg_512.RegTypes in { +defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; +} defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; @@ -821,10 +849,21 @@ defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; } // End let AddedComplexity = 100 +let OtherPredicates = [HasSMemTimeInst] in { def : GCNPat < (i64 (readcyclecounter)), (S_MEMTIME) >; +} // let OtherPredicates = [HasSMemTimeInst] + +let OtherPredicates = [HasNoSMemTimeInst] in { +def : GCNPat < + (i64 (readcyclecounter)), + (REG_SEQUENCE SReg_64, + (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, + (S_MOV_B32 (i32 0)), sub1) +>; +} // let OtherPredicates = [HasNoSMemTimeInst] //===----------------------------------------------------------------------===// // GFX10. @@ -844,7 +883,7 @@ class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> : let Inst{16} = !if(ps.has_glc, glc, ?); let Inst{25-18} = op; let Inst{31-26} = 0x3d; - let Inst{51-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{19-0}, ?), ?); + let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?); let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding), !if(ps.has_offset, offset{6-0}, ?)); } @@ -853,7 +892,7 @@ multiclass SM_Real_Loads_gfx10<bits<8> op, string ps, SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM), SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> { def _IMM_gfx10 : SMEM_Real_gfx10<op, immPs> { - let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc); + let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); } def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> { let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc, DLC:$dlc); @@ -873,7 +912,7 @@ multiclass SM_Real_Stores_gfx10<bits<8> op, string ps, // FIXME: The operand name $offset is inconsistent with $soff used // in the pseudo def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> { - let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc, DLC:$dlc); + let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, GLC:$glc, DLC:$dlc); } def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> { @@ -1020,3 +1059,12 @@ defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">; defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">; } // End SubtargetPredicate = HasScalarAtomics + +def SMInfoTable : GenericTable { + let FilterClass = "SM_Real"; + let CppTypeName = "SMInfo"; + let Fields = ["Opcode", "is_buffer"]; + + let PrimaryKey = ["Opcode"]; + let PrimaryKeyName = "getSMEMOpcodeHelper"; +} |