From cfca06d7963fa0909f90483b42a6d7d194d01e08 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Sun, 26 Jul 2020 19:36:28 +0000 Subject: Vendor import of llvm-project master 2e10b7a39b9, the last commit before the llvmorg-12-init tag, from which release/11.x was branched. --- .../Target/WebAssembly/WebAssemblyInstrAtomics.td | 837 +++++++++++++-------- 1 file changed, 507 insertions(+), 330 deletions(-) (limited to 'llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td') diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td index a9a99d38f9f1..256b77e33db9 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td @@ -13,10 +13,11 @@ let UseNamedOperandTable = 1 in multiclass ATOMIC_I pattern_r, string asmstr_r = "", - string asmstr_s = "", bits<32> atomic_op = -1> { + list pattern_r, string asmstr_r, + string asmstr_s, bits<32> atomic_op, + string is64 = "false"> { defm "" : I, + !or(0xfe00, !and(0xff, atomic_op)), is64>, Requires<[HasAtomics]>; } @@ -32,85 +33,166 @@ multiclass ATOMIC_NRI pattern, string asmstr = "", //===----------------------------------------------------------------------===// let hasSideEffects = 1 in { -defm ATOMIC_NOTIFY : +defm ATOMIC_NOTIFY_A32 : ATOMIC_I<(outs I32:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), (outs), (ins P2Align:$p2align, offset32_op:$off), [], "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", - "atomic.notify \t${off}${p2align}", 0x00>; + "atomic.notify \t${off}${p2align}", 0x00, "false">; +defm ATOMIC_NOTIFY_A64 : + ATOMIC_I<(outs I32:$dst), + (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), + (outs), (ins P2Align:$p2align, offset64_op:$off), [], + "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", + "atomic.notify \t${off}${p2align}", 0x00, "true">; let mayLoad = 1 in { -defm ATOMIC_WAIT_I32 : +defm ATOMIC_WAIT_I32_A32 : ATOMIC_I<(outs I32:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, I64:$timeout), (outs), (ins P2Align:$p2align, offset32_op:$off), [], "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", - "i32.atomic.wait \t${off}${p2align}", 0x01>; -defm ATOMIC_WAIT_I64 : + "i32.atomic.wait \t${off}${p2align}", 0x01, "false">; +defm ATOMIC_WAIT_I32_A64 : + ATOMIC_I<(outs I32:$dst), + (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, + I64:$timeout), + (outs), (ins P2Align:$p2align, offset64_op:$off), [], + "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", + "i32.atomic.wait \t${off}${p2align}", 0x01, "true">; +defm ATOMIC_WAIT_I64_A32 : ATOMIC_I<(outs I32:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, I64:$timeout), (outs), (ins P2Align:$p2align, offset32_op:$off), [], "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", - "i64.atomic.wait \t${off}${p2align}", 0x02>; + "i64.atomic.wait \t${off}${p2align}", 0x02, "false">; +defm ATOMIC_WAIT_I64_A64 : + ATOMIC_I<(outs I32:$dst), + (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, + I64:$timeout), + (outs), (ins P2Align:$p2align, offset64_op:$off), [], + "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", + "i64.atomic.wait \t${off}${p2align}", 0x02, "true">; } // mayLoad = 1 } // hasSideEffects = 1 let Predicates = [HasAtomics] in { // Select notifys with no constant offset. -def NotifyPatNoOffset : +def NotifyPatNoOffset_A32 : Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), - (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; + (ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, + Requires<[HasAddr32]>; +def NotifyPatNoOffset_A64 : + Pat<(i32 (int_wasm_atomic_notify I64:$addr, I32:$count)), + (ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, + Requires<[HasAddr64]>; // Select notifys with a constant offset. // Pattern with address + immediate offset -class NotifyPatImmOff : - Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), I32:$count)), - (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; -def : NotifyPatImmOff; -def : NotifyPatImmOff; +multiclass NotifyPatImmOff { + def : Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), + I32:$count)), + (!cast(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, + Requires<[HasAddr32]>; + def : Pat<(i32 (int_wasm_atomic_notify (operand I64:$addr, imm:$off), + I32:$count)), + (!cast(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, + Requires<[HasAddr64]>; +} +defm : NotifyPatImmOff; +defm : NotifyPatImmOff; // Select notifys with just a constant offset. -def NotifyPatOffsetOnly : +def NotifyPatOffsetOnly_A32 : + Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), + (ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, + Requires<[HasAddr32]>; +def NotifyPatOffsetOnly_A64 : Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), - (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; + (ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, + Requires<[HasAddr64]>; -def NotifyPatGlobalAddrOffOnly : +def NotifyPatGlobalAddrOffOnly_A32 : Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), I32:$count)), - (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; + (ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>, + Requires<[HasAddr32]>; +def NotifyPatGlobalAddrOffOnly_A64 : + Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), + I32:$count)), + (ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count)>, + Requires<[HasAddr64]>; // Select waits with no constant offset. -class WaitPatNoOffset : - Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), - (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; -def : WaitPatNoOffset; -def : WaitPatNoOffset; +multiclass WaitPatNoOffset { + def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), + (!cast(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, + Requires<[HasAddr32]>; + def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), + (!cast(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, + Requires<[HasAddr64]>; +} +defm : WaitPatNoOffset; +defm : WaitPatNoOffset; +defm : WaitPatNoOffset; +defm : WaitPatNoOffset; // Select waits with a constant offset. // Pattern with address + immediate offset -class WaitPatImmOff : - Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), - (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; -def : WaitPatImmOff; -def : WaitPatImmOff; -def : WaitPatImmOff; -def : WaitPatImmOff; - -// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. -class WaitPatOffsetOnly : - Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), - (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; -def : WaitPatOffsetOnly; -def : WaitPatOffsetOnly; - -class WaitPatGlobalAddrOffOnly : - Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), - (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; -def : WaitPatGlobalAddrOffOnly; -def : WaitPatGlobalAddrOffOnly; +multiclass WaitPatImmOff { + def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), + (!cast(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, + I64:$timeout)>, + Requires<[HasAddr32]>; + def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), + (!cast(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, + I64:$timeout)>, + Requires<[HasAddr64]>; +} +defm : WaitPatImmOff; +defm : WaitPatImmOff; +defm : WaitPatImmOff; +defm : WaitPatImmOff; + +// Select wait_i32, "ATOMIC_WAIT_I32s with just a constant offset. +multiclass WaitPatOffsetOnly { + def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), + (!cast(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, + I64:$timeout)>, + Requires<[HasAddr32]>; + def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), + (!cast(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, + I64:$timeout)>, + Requires<[HasAddr64]>; +} +defm : WaitPatOffsetOnly; +defm : WaitPatOffsetOnly; + +multiclass WaitPatGlobalAddrOffOnly { + def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, + I64:$timeout)), + (!cast(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, + I64:$timeout)>, + Requires<[HasAddr32]>; + def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, + I64:$timeout)), + (!cast(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, + I64:$timeout)>, + Requires<[HasAddr64]>; +} +defm : WaitPatGlobalAddrOffOnly; +defm : WaitPatGlobalAddrOffOnly; } // Predicates = [HasAtomics] //===----------------------------------------------------------------------===// @@ -131,8 +213,8 @@ defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", //===----------------------------------------------------------------------===// multiclass AtomicLoad { - defm "" : WebAssemblyLoad, - Requires<[HasAtomics]>; + defm "" : WebAssemblyLoad; } defm ATOMIC_LOAD_I32 : AtomicLoad; @@ -140,23 +222,23 @@ defm ATOMIC_LOAD_I64 : AtomicLoad; // Select loads with no constant offset. let Predicates = [HasAtomics] in { -def : LoadPatNoOffset; -def : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; // Select loads with a constant offset. // Pattern with address + immediate offset -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; // Select loads with just a constant offset. -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; } // Predicates = [HasAtomics] @@ -205,62 +287,62 @@ def sext_aload_16_64 : let Predicates = [HasAtomics] in { // Select zero-extending loads with no constant offset. -def : LoadPatNoOffset; -def : LoadPatNoOffset; -def : LoadPatNoOffset; -def : LoadPatNoOffset; -def : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; // Select sign-extending loads with no constant offset -def : LoadPatNoOffset; -def : LoadPatNoOffset; -def : LoadPatNoOffset; -def : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; +defm : LoadPatNoOffset; // 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s // Zero-extending loads with constant offset -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; // Sign-extending loads with constant offset -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; -def : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; +defm : LoadPatImmOff; // No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 // Extending loads with just a constant offset -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; -def : LoadPatOffsetOnly; - -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; -def : LoadPatGlobalAddrOffOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; +defm : LoadPatOffsetOnly; + +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; +defm : LoadPatGlobalAddrOffOnly; } // Predicates = [HasAtomics] @@ -269,8 +351,8 @@ def : LoadPatGlobalAddrOffOnly; //===----------------------------------------------------------------------===// multiclass AtomicStore { - defm "" : WebAssemblyStore, - Requires<[HasAtomics]>; + defm "" : WebAssemblyStore; } defm ATOMIC_STORE_I32 : AtomicStore; @@ -284,33 +366,54 @@ defm ATOMIC_STORE_I64 : AtomicStore; let Predicates = [HasAtomics] in { // Select stores with no constant offset. -class AStorePatNoOffset : - Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; -def : AStorePatNoOffset; -def : AStorePatNoOffset; +multiclass AStorePatNoOffset { + def : Pat<(kind I32:$addr, ty:$val), + (!cast(inst#_A32) 0, 0, I32:$addr, ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(kind I64:$addr, ty:$val), + (!cast(inst#_A64) 0, 0, I64:$addr, ty:$val)>, + Requires<[HasAddr64]>; +} +defm : AStorePatNoOffset; +defm : AStorePatNoOffset; // Select stores with a constant offset. // Pattern with address + immediate offset -class AStorePatImmOff : - Pat<(kind (operand I32:$addr, imm:$off), ty:$val), - (inst 0, imm:$off, I32:$addr, ty:$val)>; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; +multiclass AStorePatImmOff { + def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), + (!cast(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), + (!cast(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, + Requires<[HasAddr64]>; +} +defm : AStorePatImmOff; +defm : AStorePatImmOff; // Select stores with just a constant offset. -class AStorePatOffsetOnly : - Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; -def : AStorePatOffsetOnly; -def : AStorePatOffsetOnly; - -class AStorePatGlobalAddrOffOnly : - Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), - (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; -def : AStorePatGlobalAddrOffOnly; -def : AStorePatGlobalAddrOffOnly; +multiclass AStorePatOffsetOnly { + def : Pat<(kind imm:$off, ty:$val), + (!cast(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(kind imm:$off, ty:$val), + (!cast(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, + Requires<[HasAddr64]>; +} +defm : AStorePatOffsetOnly; +defm : AStorePatOffsetOnly; + +multiclass AStorePatGlobalAddrOffOnly { + def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), + (!cast(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), + (!cast(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, + Requires<[HasAddr64]>; +} +defm : AStorePatGlobalAddrOffOnly; +defm : AStorePatGlobalAddrOffOnly; } // Predicates = [HasAtomics] @@ -336,36 +439,40 @@ def trunc_astore_32_64 : trunc_astore_64; let Predicates = [HasAtomics] in { // Truncating stores with no constant offset -def : AStorePatNoOffset; -def : AStorePatNoOffset; -def : AStorePatNoOffset; -def : AStorePatNoOffset; -def : AStorePatNoOffset; +defm : AStorePatNoOffset; +defm : AStorePatNoOffset; +defm : AStorePatNoOffset; +defm : AStorePatNoOffset; +defm : AStorePatNoOffset; // Truncating stores with a constant offset -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; -def : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; +defm : AStorePatImmOff; // Truncating stores with just a constant offset -def : AStorePatOffsetOnly; -def : AStorePatOffsetOnly; -def : AStorePatOffsetOnly; -def : AStorePatOffsetOnly; -def : AStorePatOffsetOnly; - -def : AStorePatGlobalAddrOffOnly; -def : AStorePatGlobalAddrOffOnly; -def : AStorePatGlobalAddrOffOnly; -def : AStorePatGlobalAddrOffOnly; -def : AStorePatGlobalAddrOffOnly; +defm : AStorePatOffsetOnly; +defm : AStorePatOffsetOnly; +defm : AStorePatOffsetOnly; +defm : AStorePatOffsetOnly; +defm : AStorePatOffsetOnly; + +defm : AStorePatGlobalAddrOffOnly; +defm : AStorePatGlobalAddrOffOnly; +defm : AStorePatGlobalAddrOffOnly; +defm : AStorePatGlobalAddrOffOnly; +defm : AStorePatGlobalAddrOffOnly; } // Predicates = [HasAtomics] @@ -375,12 +482,18 @@ def : AStorePatGlobalAddrOffOnly; multiclass WebAssemblyBinRMW { - defm "" : + defm "_A32" : ATOMIC_I<(outs rc:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), (outs), (ins P2Align:$p2align, offset32_op:$off), [], !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), - !strconcat(name, "\t${off}${p2align}"), atomic_op>; + !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; + defm "_A64" : + ATOMIC_I<(outs rc:$dst), + (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), + (outs), (ins P2Align:$p2align, offset64_op:$off), [], + !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), + !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; } defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW; @@ -464,56 +577,78 @@ defm ATOMIC_RMW32_U_XCHG_I64 : WebAssemblyBinRMW; // Select binary RMWs with no constant offset. -class BinRMWPatNoOffset : - Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; +multiclass BinRMWPatNoOffset { + def : Pat<(ty (kind I32:$addr, ty:$val)), + (!cast(inst#_A32) 0, 0, I32:$addr, ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind I64:$addr, ty:$val)), + (!cast(inst#_A64) 0, 0, I64:$addr, ty:$val)>, + Requires<[HasAddr64]>; +} // Select binary RMWs with a constant offset. // Pattern with address + immediate offset -class BinRMWPatImmOff : - Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), - (inst 0, imm:$off, I32:$addr, ty:$val)>; +multiclass BinRMWPatImmOff { + def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), + (!cast(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), + (!cast(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, + Requires<[HasAddr64]>; +} // Select binary RMWs with just a constant offset. -class BinRMWPatOffsetOnly : - Pat<(ty (kind imm:$off, ty:$val)), - (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; +multiclass BinRMWPatOffsetOnly { + def : Pat<(ty (kind imm:$off, ty:$val)), + (!cast(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind imm:$off, ty:$val)), + (!cast(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, + Requires<[HasAddr64]>; +} -class BinRMWPatGlobalAddrOffOnly : - Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), - (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; +multiclass BinRMWPatGlobalAddrOffOnly { + def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), + (!cast(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), + (!cast(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, + Requires<[HasAddr64]>; +} // Patterns for various addressing modes. -multiclass BinRMWPattern { - def : BinRMWPatNoOffset; - def : BinRMWPatNoOffset; +multiclass BinRMWPattern { + defm : BinRMWPatNoOffset; + defm : BinRMWPatNoOffset; - def : BinRMWPatImmOff; - def : BinRMWPatImmOff; - def : BinRMWPatImmOff; - def : BinRMWPatImmOff; + defm : BinRMWPatImmOff; + defm : BinRMWPatImmOff; + defm : BinRMWPatImmOff; + defm : BinRMWPatImmOff; - def : BinRMWPatOffsetOnly; - def : BinRMWPatOffsetOnly; + defm : BinRMWPatOffsetOnly; + defm : BinRMWPatOffsetOnly; - def : BinRMWPatGlobalAddrOffOnly; - def : BinRMWPatGlobalAddrOffOnly; + defm : BinRMWPatGlobalAddrOffOnly; + defm : BinRMWPatGlobalAddrOffOnly; } let Predicates = [HasAtomics] in { -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; +defm : BinRMWPattern; +defm : BinRMWPattern; +defm : BinRMWPattern; +defm : BinRMWPattern; +defm : BinRMWPattern; +defm : BinRMWPattern; } // Predicates = [HasAtomics] // Truncating & zero-extending binary RMW patterns. @@ -556,87 +691,93 @@ multiclass BinRMWTruncExtPattern< PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { // Truncating-extending binary RMWs with no constant offset - def : BinRMWPatNoOffset, inst8_32>; - def : BinRMWPatNoOffset, inst16_32>; - def : BinRMWPatNoOffset, inst8_64>; - def : BinRMWPatNoOffset, inst16_64>; - def : BinRMWPatNoOffset, inst32_64>; + defm : BinRMWPatNoOffset, inst8_32>; + defm : BinRMWPatNoOffset, inst16_32>; + defm : BinRMWPatNoOffset, inst8_64>; + defm : BinRMWPatNoOffset, inst16_64>; + defm : BinRMWPatNoOffset, inst32_64>; - def : BinRMWPatNoOffset, inst8_32>; - def : BinRMWPatNoOffset, inst16_32>; - def : BinRMWPatNoOffset, inst8_64>; - def : BinRMWPatNoOffset, inst16_64>; + defm : BinRMWPatNoOffset, inst8_32>; + defm : BinRMWPatNoOffset, inst16_32>; + defm : BinRMWPatNoOffset, inst8_64>; + defm : BinRMWPatNoOffset, inst16_64>; // Truncating-extending binary RMWs with a constant offset - def : BinRMWPatImmOff, regPlusImm, inst8_32>; - def : BinRMWPatImmOff, regPlusImm, inst16_32>; - def : BinRMWPatImmOff, regPlusImm, inst8_64>; - def : BinRMWPatImmOff, regPlusImm, inst16_64>; - def : BinRMWPatImmOff, regPlusImm, inst32_64>; - def : BinRMWPatImmOff, or_is_add, inst8_32>; - def : BinRMWPatImmOff, or_is_add, inst16_32>; - def : BinRMWPatImmOff, or_is_add, inst8_64>; - def : BinRMWPatImmOff, or_is_add, inst16_64>; - def : BinRMWPatImmOff, or_is_add, inst32_64>; - - def : BinRMWPatImmOff, regPlusImm, inst8_32>; - def : BinRMWPatImmOff, regPlusImm, inst16_32>; - def : BinRMWPatImmOff, regPlusImm, inst8_64>; - def : BinRMWPatImmOff, regPlusImm, inst16_64>; - def : BinRMWPatImmOff, or_is_add, inst8_32>; - def : BinRMWPatImmOff, or_is_add, inst16_32>; - def : BinRMWPatImmOff, or_is_add, inst8_64>; - def : BinRMWPatImmOff, or_is_add, inst16_64>; + defm : BinRMWPatImmOff, regPlusImm, inst8_32>; + defm : BinRMWPatImmOff, regPlusImm, + inst16_32>; + defm : BinRMWPatImmOff, regPlusImm, inst8_64>; + defm : BinRMWPatImmOff, regPlusImm, + inst16_64>; + defm : BinRMWPatImmOff, regPlusImm, + inst32_64>; + defm : BinRMWPatImmOff, or_is_add, inst8_32>; + defm : BinRMWPatImmOff, or_is_add, inst16_32>; + defm : BinRMWPatImmOff, or_is_add, inst8_64>; + defm : BinRMWPatImmOff, or_is_add, inst16_64>; + defm : BinRMWPatImmOff, or_is_add, inst32_64>; + + defm : BinRMWPatImmOff, regPlusImm, inst8_32>; + defm : BinRMWPatImmOff, regPlusImm, + inst16_32>; + defm : BinRMWPatImmOff, regPlusImm, inst8_64>; + defm : BinRMWPatImmOff, regPlusImm, + inst16_64>; + defm : BinRMWPatImmOff, or_is_add, inst8_32>; + defm : BinRMWPatImmOff, or_is_add, inst16_32>; + defm : BinRMWPatImmOff, or_is_add, inst8_64>; + defm : BinRMWPatImmOff, or_is_add, inst16_64>; // Truncating-extending binary RMWs with just a constant offset - def : BinRMWPatOffsetOnly, inst8_32>; - def : BinRMWPatOffsetOnly, inst16_32>; - def : BinRMWPatOffsetOnly, inst8_64>; - def : BinRMWPatOffsetOnly, inst16_64>; - def : BinRMWPatOffsetOnly, inst32_64>; - - def : BinRMWPatOffsetOnly, inst8_32>; - def : BinRMWPatOffsetOnly, inst16_32>; - def : BinRMWPatOffsetOnly, inst8_64>; - def : BinRMWPatOffsetOnly, inst16_64>; - - def : BinRMWPatGlobalAddrOffOnly, inst8_32>; - def : BinRMWPatGlobalAddrOffOnly, inst16_32>; - def : BinRMWPatGlobalAddrOffOnly, inst8_64>; - def : BinRMWPatGlobalAddrOffOnly, inst16_64>; - def : BinRMWPatGlobalAddrOffOnly, inst32_64>; - - def : BinRMWPatGlobalAddrOffOnly, inst8_32>; - def : BinRMWPatGlobalAddrOffOnly, inst16_32>; - def : BinRMWPatGlobalAddrOffOnly, inst8_64>; - def : BinRMWPatGlobalAddrOffOnly, inst16_64>; + defm : BinRMWPatOffsetOnly, inst8_32>; + defm : BinRMWPatOffsetOnly, inst16_32>; + defm : BinRMWPatOffsetOnly, inst8_64>; + defm : BinRMWPatOffsetOnly, inst16_64>; + defm : BinRMWPatOffsetOnly, inst32_64>; + + defm : BinRMWPatOffsetOnly, inst8_32>; + defm : BinRMWPatOffsetOnly, inst16_32>; + defm : BinRMWPatOffsetOnly, inst8_64>; + defm : BinRMWPatOffsetOnly, inst16_64>; + + defm : BinRMWPatGlobalAddrOffOnly, inst8_32>; + defm : BinRMWPatGlobalAddrOffOnly, inst16_32>; + defm : BinRMWPatGlobalAddrOffOnly, inst8_64>; + defm : BinRMWPatGlobalAddrOffOnly, inst16_64>; + defm : BinRMWPatGlobalAddrOffOnly, inst32_64>; + + defm : BinRMWPatGlobalAddrOffOnly, inst8_32>; + defm : BinRMWPatGlobalAddrOffOnly, inst16_32>; + defm : BinRMWPatGlobalAddrOffOnly, inst8_64>; + defm : BinRMWPatGlobalAddrOffOnly, inst16_64>; } let Predicates = [HasAtomics] in { defm : BinRMWTruncExtPattern< atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, - ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, - ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; + "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", + "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; defm : BinRMWTruncExtPattern< atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, - ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, - ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; + "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", + "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; defm : BinRMWTruncExtPattern< atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, - ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, - ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; + "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", + "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; defm : BinRMWTruncExtPattern< atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, - ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, - ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; + "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", + "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; defm : BinRMWTruncExtPattern< atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, - ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, - ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; + "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", + "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; defm : BinRMWTruncExtPattern< atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, - ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, - ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; + "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", + "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", + "ATOMIC_RMW32_U_XCHG_I64">; } // Predicates = [HasAtomics] //===----------------------------------------------------------------------===// @@ -651,13 +792,20 @@ defm : BinRMWTruncExtPattern< multiclass WebAssemblyTerRMW { - defm "" : + defm "_A32" : ATOMIC_I<(outs rc:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, rc:$new_), (outs), (ins P2Align:$p2align, offset32_op:$off), [], !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), - !strconcat(name, "\t${off}${p2align}"), atomic_op>; + !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; + defm "_A64" : + ATOMIC_I<(outs rc:$dst), + (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, + rc:$new_), + (outs), (ins P2Align:$p2align, offset64_op:$off), [], + !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), + !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; } defm ATOMIC_RMW_CMPXCHG_I32 : @@ -676,47 +824,70 @@ defm ATOMIC_RMW32_U_CMPXCHG_I64 : WebAssemblyTerRMW; // Select ternary RMWs with no constant offset. -class TerRMWPatNoOffset : - Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), - (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; +multiclass TerRMWPatNoOffset { + def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), + (!cast(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), + (!cast(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, + Requires<[HasAddr64]>; +} // Select ternary RMWs with a constant offset. // Pattern with address + immediate offset -class TerRMWPatImmOff : - Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), - (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; +multiclass TerRMWPatImmOff { + def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), + (!cast(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), + (!cast(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, + Requires<[HasAddr64]>; +} // Select ternary RMWs with just a constant offset. -class TerRMWPatOffsetOnly : - Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), - (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; +multiclass TerRMWPatOffsetOnly { + def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), + (!cast(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, + ty:$new)>; + def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), + (!cast(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, + ty:$new)>; +} -class TerRMWPatGlobalAddrOffOnly : - Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), - (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; +multiclass TerRMWPatGlobalAddrOffOnly { + def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), + (!cast(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, + ty:$new)>, + Requires<[HasAddr32]>; + def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), + (!cast(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, + ty:$new)>, + Requires<[HasAddr64]>; +} // Patterns for various addressing modes. -multiclass TerRMWPattern { - def : TerRMWPatNoOffset; - def : TerRMWPatNoOffset; +multiclass TerRMWPattern { + defm : TerRMWPatNoOffset; + defm : TerRMWPatNoOffset; - def : TerRMWPatImmOff; - def : TerRMWPatImmOff; - def : TerRMWPatImmOff; - def : TerRMWPatImmOff; + defm : TerRMWPatImmOff; + defm : TerRMWPatImmOff; + defm : TerRMWPatImmOff; + defm : TerRMWPatImmOff; - def : TerRMWPatOffsetOnly; - def : TerRMWPatOffsetOnly; + defm : TerRMWPatOffsetOnly; + defm : TerRMWPatOffsetOnly; - def : TerRMWPatGlobalAddrOffOnly; - def : TerRMWPatGlobalAddrOffOnly; + defm : TerRMWPatGlobalAddrOffOnly; + defm : TerRMWPatGlobalAddrOffOnly; } let Predicates = [HasAtomics] in defm : TerRMWPattern; + "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; // Truncating & zero-extending ternary RMW patterns. // DAG legalization & optimization before instruction selection may introduce @@ -759,67 +930,73 @@ class sext_ter_rmw_16_64 : sext_ter_rmw_8_64; // Patterns for various addressing modes for truncating-extending ternary RMWs. multiclass TerRMWTruncExtPattern< PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, - NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { + string inst8_32, string inst16_32, string inst8_64, string inst16_64, + string inst32_64> { // Truncating-extending ternary RMWs with no constant offset - def : TerRMWPatNoOffset, inst8_32>; - def : TerRMWPatNoOffset, inst16_32>; - def : TerRMWPatNoOffset, inst8_64>; - def : TerRMWPatNoOffset, inst16_64>; - def : TerRMWPatNoOffset, inst32_64>; + defm : TerRMWPatNoOffset, inst8_32>; + defm : TerRMWPatNoOffset, inst16_32>; + defm : TerRMWPatNoOffset, inst8_64>; + defm : TerRMWPatNoOffset, inst16_64>; + defm : TerRMWPatNoOffset, inst32_64>; - def : TerRMWPatNoOffset, inst8_32>; - def : TerRMWPatNoOffset, inst16_32>; - def : TerRMWPatNoOffset, inst8_64>; - def : TerRMWPatNoOffset, inst16_64>; + defm : TerRMWPatNoOffset, inst8_32>; + defm : TerRMWPatNoOffset, inst16_32>; + defm : TerRMWPatNoOffset, inst8_64>; + defm : TerRMWPatNoOffset, inst16_64>; // Truncating-extending ternary RMWs with a constant offset - def : TerRMWPatImmOff, regPlusImm, inst8_32>; - def : TerRMWPatImmOff, regPlusImm, inst16_32>; - def : TerRMWPatImmOff, regPlusImm, inst8_64>; - def : TerRMWPatImmOff, regPlusImm, inst16_64>; - def : TerRMWPatImmOff, regPlusImm, inst32_64>; - def : TerRMWPatImmOff, or_is_add, inst8_32>; - def : TerRMWPatImmOff, or_is_add, inst16_32>; - def : TerRMWPatImmOff, or_is_add, inst8_64>; - def : TerRMWPatImmOff, or_is_add, inst16_64>; - def : TerRMWPatImmOff, or_is_add, inst32_64>; - - def : TerRMWPatImmOff, regPlusImm, inst8_32>; - def : TerRMWPatImmOff, regPlusImm, inst16_32>; - def : TerRMWPatImmOff, regPlusImm, inst8_64>; - def : TerRMWPatImmOff, regPlusImm, inst16_64>; - def : TerRMWPatImmOff, or_is_add, inst8_32>; - def : TerRMWPatImmOff, or_is_add, inst16_32>; - def : TerRMWPatImmOff, or_is_add, inst8_64>; - def : TerRMWPatImmOff, or_is_add, inst16_64>; + defm : TerRMWPatImmOff, regPlusImm, inst8_32>; + defm : TerRMWPatImmOff, regPlusImm, + inst16_32>; + defm : TerRMWPatImmOff, regPlusImm, inst8_64>; + defm : TerRMWPatImmOff, regPlusImm, + inst16_64>; + defm : TerRMWPatImmOff, regPlusImm, + inst32_64>; + defm : TerRMWPatImmOff, or_is_add, inst8_32>; + defm : TerRMWPatImmOff, or_is_add, inst16_32>; + defm : TerRMWPatImmOff, or_is_add, inst8_64>; + defm : TerRMWPatImmOff, or_is_add, inst16_64>; + defm : TerRMWPatImmOff, or_is_add, inst32_64>; + + defm : TerRMWPatImmOff, regPlusImm, inst8_32>; + defm : TerRMWPatImmOff, regPlusImm, + inst16_32>; + defm : TerRMWPatImmOff, regPlusImm, inst8_64>; + defm : TerRMWPatImmOff, regPlusImm, + inst16_64>; + defm : TerRMWPatImmOff, or_is_add, inst8_32>; + defm : TerRMWPatImmOff, or_is_add, inst16_32>; + defm : TerRMWPatImmOff, or_is_add, inst8_64>; + defm : TerRMWPatImmOff, or_is_add, inst16_64>; // Truncating-extending ternary RMWs with just a constant offset - def : TerRMWPatOffsetOnly, inst8_32>; - def : TerRMWPatOffsetOnly, inst16_32>; - def : TerRMWPatOffsetOnly, inst8_64>; - def : TerRMWPatOffsetOnly, inst16_64>; - def : TerRMWPatOffsetOnly, inst32_64>; - - def : TerRMWPatOffsetOnly, inst8_32>; - def : TerRMWPatOffsetOnly, inst16_32>; - def : TerRMWPatOffsetOnly, inst8_64>; - def : TerRMWPatOffsetOnly, inst16_64>; - - def : TerRMWPatGlobalAddrOffOnly, inst8_32>; - def : TerRMWPatGlobalAddrOffOnly, inst16_32>; - def : TerRMWPatGlobalAddrOffOnly, inst8_64>; - def : TerRMWPatGlobalAddrOffOnly, inst16_64>; - def : TerRMWPatGlobalAddrOffOnly, inst32_64>; - - def : TerRMWPatGlobalAddrOffOnly, inst8_32>; - def : TerRMWPatGlobalAddrOffOnly, inst16_32>; - def : TerRMWPatGlobalAddrOffOnly, inst8_64>; - def : TerRMWPatGlobalAddrOffOnly, inst16_64>; + defm : TerRMWPatOffsetOnly, inst8_32>; + defm : TerRMWPatOffsetOnly, inst16_32>; + defm : TerRMWPatOffsetOnly, inst8_64>; + defm : TerRMWPatOffsetOnly, inst16_64>; + defm : TerRMWPatOffsetOnly, inst32_64>; + + defm : TerRMWPatOffsetOnly, inst8_32>; + defm : TerRMWPatOffsetOnly, inst16_32>; + defm : TerRMWPatOffsetOnly, inst8_64>; + defm : TerRMWPatOffsetOnly, inst16_64>; + + defm : TerRMWPatGlobalAddrOffOnly, inst8_32>; + defm : TerRMWPatGlobalAddrOffOnly, inst16_32>; + defm : TerRMWPatGlobalAddrOffOnly, inst8_64>; + defm : TerRMWPatGlobalAddrOffOnly, inst16_64>; + defm : TerRMWPatGlobalAddrOffOnly, inst32_64>; + + defm : TerRMWPatGlobalAddrOffOnly, inst8_32>; + defm : TerRMWPatGlobalAddrOffOnly, inst16_32>; + defm : TerRMWPatGlobalAddrOffOnly, inst8_64>; + defm : TerRMWPatGlobalAddrOffOnly, inst16_64>; } let Predicates = [HasAtomics] in defm : TerRMWTruncExtPattern< atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, - ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, - ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, - ATOMIC_RMW32_U_CMPXCHG_I64>; + "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", + "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", + "ATOMIC_RMW32_U_CMPXCHG_I64">; -- cgit v1.2.3