diff options
Diffstat (limited to 'lib/Target/WebAssembly/WebAssemblyInstrAtomics.td')
| -rw-r--r-- | lib/Target/WebAssembly/WebAssemblyInstrAtomics.td | 448 |
1 files changed, 405 insertions, 43 deletions
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td index d879932b3232..5fb8ef90bc43 100644 --- a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td +++ b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td @@ -16,10 +16,16 @@ // Atomic loads //===----------------------------------------------------------------------===// -let Defs = [ARGUMENTS] in { +multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, + list<dag> pattern_r, string asmstr_r = "", + string asmstr_s = "", bits<32> inst = -1> { + defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, + inst>, + Requires<[HasAtomics]>; +} + defm ATOMIC_LOAD_I32 : WebAssemblyLoad<I32, "i32.atomic.load", 0xfe10>; defm ATOMIC_LOAD_I64 : WebAssemblyLoad<I64, "i64.atomic.load", 0xfe11>; -} // Defs = [ARGUMENTS] // Select loads with no constant offset. let Predicates = [HasAtomics] in { @@ -54,13 +60,11 @@ def : LoadPatExternSymOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; // Extending loads. Note that there are only zero-extending atomic loads, no // sign-extending loads. -let Defs = [ARGUMENTS] in { defm ATOMIC_LOAD8_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load8_u", 0xfe12>; defm ATOMIC_LOAD16_U_I32 : WebAssemblyLoad<I32, "i32.atomic.load16_u", 0xfe13>; defm ATOMIC_LOAD8_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load8_u", 0xfe14>; defm ATOMIC_LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load16_u", 0xfe15>; defm ATOMIC_LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.atomic.load32_u", 0xfe16>; -} // Defs = [ARGUMENTS] // Fragments for extending loads. These are different from regular loads because // the SDNodes are derived from AtomicSDNode rather than LoadSDNode and @@ -110,7 +114,7 @@ def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; -// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32 +// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s // Zero-extending loads with constant offset def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; @@ -192,10 +196,8 @@ def : LoadPatExternSymOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; // Atomic stores //===----------------------------------------------------------------------===// -let Defs = [ARGUMENTS] in { defm ATOMIC_STORE_I32 : WebAssemblyStore<I32, "i32.atomic.store", 0xfe17>; defm ATOMIC_STORE_I64 : WebAssemblyStore<I64, "i64.atomic.store", 0xfe18>; -} // Defs = [ARGUMENTS] // We need an 'atomic' version of store patterns because store and atomic_store // nodes have different operand orders: @@ -255,13 +257,11 @@ def : AStorePatExternSymOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; } // Predicates = [HasAtomics] // Truncating stores. -let Defs = [ARGUMENTS] in { defm ATOMIC_STORE8_I32 : WebAssemblyStore<I32, "i32.atomic.store8", 0xfe19>; defm ATOMIC_STORE16_I32 : WebAssemblyStore<I32, "i32.atomic.store16", 0xfe1a>; defm ATOMIC_STORE8_I64 : WebAssemblyStore<I64, "i64.atomic.store8", 0xfe1b>; defm ATOMIC_STORE16_I64 : WebAssemblyStore<I64, "i64.atomic.store16", 0xfe1c>; defm ATOMIC_STORE32_I64 : WebAssemblyStore<I64, "i64.atomic.store32", 0xfe1d>; -} // Defs = [ARGUMENTS] // Fragments for truncating stores. @@ -333,8 +333,6 @@ def : AStorePatExternSymOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; // Atomic binary read-modify-writes //===----------------------------------------------------------------------===// -let Defs = [ARGUMENTS] in { - multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> { defm "" : I<(outs rc:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), @@ -346,83 +344,82 @@ multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> { defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>; defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>; defm ATOMIC_RMW8_U_ADD_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.add", 0xfe20>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0xfe20>; defm ATOMIC_RMW16_U_ADD_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.add", 0xfe21>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0xfe21>; defm ATOMIC_RMW8_U_ADD_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.add", 0xfe22>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0xfe22>; defm ATOMIC_RMW16_U_ADD_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.add", 0xfe23>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0xfe23>; defm ATOMIC_RMW32_U_ADD_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.add", 0xfe24>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0xfe24>; defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>; defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>; defm ATOMIC_RMW8_U_SUB_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.sub", 0xfe27>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0xfe27>; defm ATOMIC_RMW16_U_SUB_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.sub", 0xfe28>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0xfe28>; defm ATOMIC_RMW8_U_SUB_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.sub", 0xfe29>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0xfe29>; defm ATOMIC_RMW16_U_SUB_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.sub", 0xfe2a>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0xfe2a>; defm ATOMIC_RMW32_U_SUB_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.sub", 0xfe2b>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0xfe2b>; defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>; defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>; defm ATOMIC_RMW8_U_AND_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.and", 0xfe2e>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0xfe2e>; defm ATOMIC_RMW16_U_AND_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.and", 0xfe2f>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0xfe2f>; defm ATOMIC_RMW8_U_AND_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.and", 0xfe30>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0xfe30>; defm ATOMIC_RMW16_U_AND_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.and", 0xfe31>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0xfe31>; defm ATOMIC_RMW32_U_AND_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.and", 0xfe32>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0xfe32>; defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>; defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>; defm ATOMIC_RMW8_U_OR_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.or", 0xfe35>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0xfe35>; defm ATOMIC_RMW16_U_OR_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.or", 0xfe36>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0xfe36>; defm ATOMIC_RMW8_U_OR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.or", 0xfe37>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0xfe37>; defm ATOMIC_RMW16_U_OR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.or", 0xfe38>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0xfe38>; defm ATOMIC_RMW32_U_OR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.or", 0xfe39>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0xfe39>; defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>; defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>; defm ATOMIC_RMW8_U_XOR_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xor", 0xfe3c>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0xfe3c>; defm ATOMIC_RMW16_U_XOR_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xor", 0xfe3d>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0xfe3d>; defm ATOMIC_RMW8_U_XOR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xor", 0xfe3e>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0xfe3e>; defm ATOMIC_RMW16_U_XOR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xor", 0xfe3f>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0xfe3f>; defm ATOMIC_RMW32_U_XOR_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xor", 0xfe40>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0xfe40>; defm ATOMIC_RMW_XCHG_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>; defm ATOMIC_RMW_XCHG_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>; defm ATOMIC_RMW8_U_XCHG_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xchg", 0xfe43>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0xfe43>; defm ATOMIC_RMW16_U_XCHG_I32 : - WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xchg", 0xfe44>; + WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0xfe44>; defm ATOMIC_RMW8_U_XCHG_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xchg", 0xfe45>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0xfe45>; defm ATOMIC_RMW16_U_XCHG_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xchg", 0xfe46>; + WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0xfe46>; defm ATOMIC_RMW32_U_XCHG_I64 : - WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xchg", 0xfe47>; -} + WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0xfe47>; // Select binary RMWs with no constant offset. class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : @@ -533,7 +530,7 @@ class sext_bin_rmw_8_64<PatFrag kind> : PatFrag<(ops node:$addr, node:$val), (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; -// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32 +// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s // Patterns for various addressing modes for truncating-extending binary RMWs. multiclass BinRMWTruncExtPattern< @@ -655,3 +652,368 @@ defm : BinRMWTruncExtPattern< ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; } // Predicates = [HasAtomics] + +//===----------------------------------------------------------------------===// +// Atomic ternary read-modify-writes +//===----------------------------------------------------------------------===// + +// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success +// flag}. When we use the success flag or both values, we can't make use of i64 +// truncate/extend versions of instructions for now, which is suboptimal. +// Consider adding a pass after instruction selection that optimizes this case +// if it is frequent. + +multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string Name, int Opcode> { + defm "" : I<(outs rc:$dst), + (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, + rc:$new), + (outs), (ins P2Align:$p2align, offset32_op:$off), [], + !strconcat(Name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new"), + !strconcat(Name, "\t${off}, ${p2align}"), Opcode>; +} + +defm ATOMIC_RMW_CMPXCHG_I32 : + WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0xfe48>; +defm ATOMIC_RMW_CMPXCHG_I64 : + WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>; +defm ATOMIC_RMW8_U_CMPXCHG_I32 : + WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0xfe4a>; +defm ATOMIC_RMW16_U_CMPXCHG_I32 : + WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0xfe4b>; +defm ATOMIC_RMW8_U_CMPXCHG_I64 : + WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0xfe4c>; +defm ATOMIC_RMW16_U_CMPXCHG_I64 : + WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0xfe4d>; +defm ATOMIC_RMW32_U_CMPXCHG_I64 : + WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0xfe4e>; + +// Select ternary RMWs with no constant offset. +class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), + (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; + +// Select ternary RMWs with a constant offset. + +// Pattern with address + immediate offset +class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : + Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), + (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; + +class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), + ty:$exp, ty:$new)), + (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>; + +class TerRMWPatExternalSym<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), + ty:$exp, ty:$new)), + (inst 0, texternalsym:$off, I32:$addr, ty:$exp, ty:$new)>; + +// Select ternary RMWs with just a constant offset. +class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), + (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; + +class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), + (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; + +class TerRMWPatExternSymOffOnly<ValueType ty, PatFrag kind, NI inst> : + Pat<(ty (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, ty:$new)), + (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, ty:$new)>; + +// Patterns for various addressing modes. +multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, + NI inst_64> { + def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; + def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; + + def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; + def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; + def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; + def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; + + def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>; + def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>; + + def : TerRMWPatExternalSym<i32, rmw_32, inst_32>; + def : TerRMWPatExternalSym<i64, rmw_64, inst_64>; + + def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; + def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; + + def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; + def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; + + def : TerRMWPatExternSymOffOnly<i32, rmw_32, inst_32>; + def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>; +} + +let Predicates = [HasAtomics] in { +defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, + ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; +} // Predicates = [HasAtomics] + +// Truncating & zero-extending ternary RMW patterns. +// DAG legalization & optimization before instruction selection may introduce +// additional nodes such as anyext or assertzext depending on operand types. +class zext_ter_rmw_8_32<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; +class zext_ter_rmw_16_32<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; +class zext_ter_rmw_8_64<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (zext (i32 (assertzext (i32 (kind node:$addr, + (i32 (trunc (i64 node:$exp))), + (i32 (trunc (i64 node:$new))))))))>; +class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; +class zext_ter_rmw_32_64<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (zext (i32 (kind node:$addr, + (i32 (trunc (i64 node:$exp))), + (i32 (trunc (i64 node:$new))))))>; + +// Truncating & sign-extending ternary RMW patterns. +// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a +// zext RMW; the next instruction will be sext_inreg which is selected by +// itself. +class sext_ter_rmw_8_32<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (kind node:$addr, node:$exp, node:$new)>; +class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; +class sext_ter_rmw_8_64<PatFrag kind> : + PatFrag<(ops node:$addr, node:$exp, node:$new), + (anyext (i32 (assertzext (i32 + (kind node:$addr, + (i32 (trunc (i64 node:$exp))), + (i32 (trunc (i64 node:$new))))))))>; +class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; +// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s + +// Patterns for various addressing modes for truncating-extending ternary RMWs. +multiclass TerRMWTruncExtPattern< + PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, + NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { + // Truncating-extending ternary RMWs with no constant offset + def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; + + // Truncating-extending ternary RMWs with a constant offset + def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; + def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; + def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; + def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; + def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; + + def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; + def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; + def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; + def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; + def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; + def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; + def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; + def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; + + def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; + + def : TerRMWPatExternalSym<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatExternalSym<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatExternalSym<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatExternalSym<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatExternalSym<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatExternalSym<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatExternalSym<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatExternalSym<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatExternalSym<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; + + // Truncating-extending ternary RMWs with just a constant offset + def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; + + def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; + + def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatExternSymOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; + def : TerRMWPatExternSymOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; + + def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; + def : TerRMWPatExternSymOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; + def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; + def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; +} + +let Predicates = [HasAtomics] in { +defm : TerRMWTruncExtPattern< + atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, + ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, + ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, + ATOMIC_RMW32_U_CMPXCHG_I64>; +} + +//===----------------------------------------------------------------------===// +// Atomic wait / notify +//===----------------------------------------------------------------------===// + +let hasSideEffects = 1 in { +defm ATOMIC_NOTIFY : + I<(outs I32:$dst), + (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), + (outs), (ins P2Align:$p2align, offset32_op:$off), [], + "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", + "atomic.notify \t${off}, ${p2align}", 0xfe00>; +let mayLoad = 1 in { +defm ATOMIC_WAIT_I32 : + I<(outs I32:$dst), + (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout), + (outs), (ins P2Align:$p2align, offset32_op:$off), [], + "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", + "i32.atomic.wait \t${off}, ${p2align}", 0xfe01>; +defm ATOMIC_WAIT_I64 : + I<(outs I32:$dst), + (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout), + (outs), (ins P2Align:$p2align, offset32_op:$off), [], + "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", + "i64.atomic.wait \t${off}, ${p2align}", 0xfe02>; +} // mayLoad = 1 +} // hasSideEffects = 1 + +let Predicates = [HasAtomics] in { +// Select notifys with no constant offset. +class NotifyPatNoOffset<Intrinsic kind> : + Pat<(i32 (kind I32:$addr, I32:$count)), + (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; +def : NotifyPatNoOffset<int_wasm_atomic_notify>; + +// Select notifys with a constant offset. + +// Pattern with address + immediate offset +class NotifyPatImmOff<Intrinsic kind, PatFrag operand> : + Pat<(i32 (kind (operand I32:$addr, imm:$off), I32:$count)), + (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; +def : NotifyPatImmOff<int_wasm_atomic_notify, regPlusImm>; +def : NotifyPatImmOff<int_wasm_atomic_notify, or_is_add>; + +class NotifyPatGlobalAddr<Intrinsic kind> : + Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), + I32:$count)), + (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>; +def : NotifyPatGlobalAddr<int_wasm_atomic_notify>; + +class NotifyPatExternalSym<Intrinsic kind> : + Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), + I32:$count)), + (ATOMIC_NOTIFY 0, texternalsym:$off, I32:$addr, I32:$count)>; +def : NotifyPatExternalSym<int_wasm_atomic_notify>; + +// Select notifys with just a constant offset. +class NotifyPatOffsetOnly<Intrinsic kind> : + Pat<(i32 (kind imm:$off, I32:$count)), + (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; +def : NotifyPatOffsetOnly<int_wasm_atomic_notify>; + +class NotifyPatGlobalAddrOffOnly<Intrinsic kind> : + Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), I32:$count)), + (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; +def : NotifyPatGlobalAddrOffOnly<int_wasm_atomic_notify>; + +class NotifyPatExternSymOffOnly<Intrinsic kind> : + Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), I32:$count)), + (ATOMIC_NOTIFY 0, texternalsym:$off, (CONST_I32 0), I32:$count)>; +def : NotifyPatExternSymOffOnly<int_wasm_atomic_notify>; + +// Select waits with no constant offset. +class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), + (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; +def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; + +// Select waits with a constant offset. + +// Pattern with address + immediate offset +class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : + Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), + (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; +def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; +def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; +def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; +def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; + +class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)), + ty:$exp, I64:$timeout)), + (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>; +def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; + +class WaitPatExternalSym<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind (add I32:$addr, (WebAssemblywrapper texternalsym:$off)), + ty:$exp, I64:$timeout)), + (inst 0, texternalsym:$off, I32:$addr, ty:$exp, I64:$timeout)>; +def : WaitPatExternalSym<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatExternalSym<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; + +// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. +class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), + (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; +def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; + +class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), + (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; +def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; + +class WaitPatExternSymOffOnly<ValueType ty, Intrinsic kind, NI inst> : + Pat<(i32 (kind (WebAssemblywrapper texternalsym:$off), ty:$exp, + I64:$timeout)), + (inst 0, texternalsym:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; +def : WaitPatExternSymOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; +def : WaitPatExternSymOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; +} // Predicates = [HasAtomics] |