diff options
Diffstat (limited to 'llvm/lib/Target/ARM/ARMISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/ARM/ARMISelLowering.cpp | 2579 |
1 files changed, 2016 insertions, 563 deletions
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 397979b4ab1e..900113244e41 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -21,6 +21,7 @@ #include "ARMRegisterInfo.h" #include "ARMSelectionDAGInfo.h" #include "ARMSubtarget.h" +#include "ARMTargetTransformInfo.h" #include "MCTargetDesc/ARMAddressingModes.h" #include "MCTargetDesc/ARMBaseInfo.h" #include "Utils/ARMBaseInfo.h" @@ -153,8 +154,7 @@ static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 }; -void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, - MVT PromotedBitwiseVT) { +void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) { if (VT != PromotedLdStVT) { setOperationAction(ISD::LOAD, VT, Promote); AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); @@ -193,16 +193,6 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, setOperationAction(ISD::SRL, VT, Custom); } - // Promote all bit-wise operations. - if (VT.isInteger() && VT != PromotedBitwiseVT) { - setOperationAction(ISD::AND, VT, Promote); - AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT); - setOperationAction(ISD::OR, VT, Promote); - AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT); - setOperationAction(ISD::XOR, VT, Promote); - AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT); - } - // Neon does not support vector divide/remainder operations. setOperationAction(ISD::SDIV, VT, Expand); setOperationAction(ISD::UDIV, VT, Expand); @@ -224,12 +214,12 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, void ARMTargetLowering::addDRTypeForNEON(MVT VT) { addRegisterClass(VT, &ARM::DPRRegClass); - addTypeForNEON(VT, MVT::f64, MVT::v2i32); + addTypeForNEON(VT, MVT::f64); } void ARMTargetLowering::addQRTypeForNEON(MVT VT) { addRegisterClass(VT, &ARM::DPairRegClass); - addTypeForNEON(VT, MVT::v2f64, MVT::v4i32); + addTypeForNEON(VT, MVT::v2f64); } void ARMTargetLowering::setAllExpand(MVT VT) { @@ -280,6 +270,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::UADDSAT, VT, Legal); setOperationAction(ISD::SSUBSAT, VT, Legal); setOperationAction(ISD::USUBSAT, VT, Legal); + setOperationAction(ISD::ABDS, VT, Legal); + setOperationAction(ISD::ABDU, VT, Legal); // No native support for these. setOperationAction(ISD::UDIV, VT, Expand); @@ -396,6 +388,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); } + setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); + // We can do bitwise operations on v2i64 vectors setOperationAction(ISD::AND, MVT::v2i64, Legal); setOperationAction(ISD::OR, MVT::v2i64, Legal); @@ -447,6 +441,14 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::VSELECT, VT, Expand); setOperationAction(ISD::SELECT, VT, Expand); } + setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); + setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); + setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); + setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); + setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); + setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); + setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); + setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); } ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, @@ -541,6 +543,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setLibcallName(RTLIB::SHL_I128, nullptr); setLibcallName(RTLIB::SRL_I128, nullptr); setLibcallName(RTLIB::SRA_I128, nullptr); + setLibcallName(RTLIB::MUL_I128, nullptr); // RTLIB if (Subtarget->isAAPCS_ABI() && @@ -766,9 +769,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, addAllExtLoads(VT, InnerVT, Expand); } - setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); - setOperationAction(ISD::MULHU, VT, Expand); setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::BSWAP, VT, Expand); @@ -947,6 +948,11 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) { + setOperationAction(ISD::MULHS, VT, Expand); + setOperationAction(ISD::MULHU, VT, Expand); + } + // NEON only has FMA instructions as of VFP4. if (!Subtarget->hasVFP4Base()) { setOperationAction(ISD::FMA, MVT::v2f32, Expand); @@ -977,6 +983,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setTargetDAGCombine(ISD::VECTOR_SHUFFLE); setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); + setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); setTargetDAGCombine(ISD::STORE); setTargetDAGCombine(ISD::SIGN_EXTEND); setTargetDAGCombine(ISD::ZERO_EXTEND); @@ -1103,6 +1110,10 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SSUBSAT, MVT::i8, Custom); setOperationAction(ISD::SADDSAT, MVT::i16, Custom); setOperationAction(ISD::SSUBSAT, MVT::i16, Custom); + setOperationAction(ISD::UADDSAT, MVT::i8, Custom); + setOperationAction(ISD::USUBSAT, MVT::i8, Custom); + setOperationAction(ISD::UADDSAT, MVT::i16, Custom); + setOperationAction(ISD::USUBSAT, MVT::i16, Custom); } if (Subtarget->hasBaseDSP()) { setOperationAction(ISD::SADDSAT, MVT::i32, Legal); @@ -1340,6 +1351,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM, // iff target supports vfp2. setOperationAction(ISD::BITCAST, MVT::i64, Custom); setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); + setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); } // We want to custom lower some of our intrinsics. @@ -1604,209 +1616,215 @@ ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, } const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { +#define MAKE_CASE(V) \ + case V: \ + return #V; switch ((ARMISD::NodeType)Opcode) { - case ARMISD::FIRST_NUMBER: break; - case ARMISD::Wrapper: return "ARMISD::Wrapper"; - case ARMISD::WrapperPIC: return "ARMISD::WrapperPIC"; - case ARMISD::WrapperJT: return "ARMISD::WrapperJT"; - case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL"; - case ARMISD::CALL: return "ARMISD::CALL"; - case ARMISD::CALL_PRED: return "ARMISD::CALL_PRED"; - case ARMISD::CALL_NOLINK: return "ARMISD::CALL_NOLINK"; - case ARMISD::tSECALL: return "ARMISD::tSECALL"; - case ARMISD::BRCOND: return "ARMISD::BRCOND"; - case ARMISD::BR_JT: return "ARMISD::BR_JT"; - case ARMISD::BR2_JT: return "ARMISD::BR2_JT"; - case ARMISD::RET_FLAG: return "ARMISD::RET_FLAG"; - case ARMISD::SERET_FLAG: return "ARMISD::SERET_FLAG"; - case ARMISD::INTRET_FLAG: return "ARMISD::INTRET_FLAG"; - case ARMISD::PIC_ADD: return "ARMISD::PIC_ADD"; - case ARMISD::CMP: return "ARMISD::CMP"; - case ARMISD::CMN: return "ARMISD::CMN"; - case ARMISD::CMPZ: return "ARMISD::CMPZ"; - case ARMISD::CMPFP: return "ARMISD::CMPFP"; - case ARMISD::CMPFPE: return "ARMISD::CMPFPE"; - case ARMISD::CMPFPw0: return "ARMISD::CMPFPw0"; - case ARMISD::CMPFPEw0: return "ARMISD::CMPFPEw0"; - case ARMISD::BCC_i64: return "ARMISD::BCC_i64"; - case ARMISD::FMSTAT: return "ARMISD::FMSTAT"; - - case ARMISD::CMOV: return "ARMISD::CMOV"; - case ARMISD::SUBS: return "ARMISD::SUBS"; - - case ARMISD::SSAT: return "ARMISD::SSAT"; - case ARMISD::USAT: return "ARMISD::USAT"; - - case ARMISD::ASRL: return "ARMISD::ASRL"; - case ARMISD::LSRL: return "ARMISD::LSRL"; - case ARMISD::LSLL: return "ARMISD::LSLL"; - - case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG"; - case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG"; - case ARMISD::RRX: return "ARMISD::RRX"; - - case ARMISD::ADDC: return "ARMISD::ADDC"; - case ARMISD::ADDE: return "ARMISD::ADDE"; - case ARMISD::SUBC: return "ARMISD::SUBC"; - case ARMISD::SUBE: return "ARMISD::SUBE"; - case ARMISD::LSLS: return "ARMISD::LSLS"; - - case ARMISD::VMOVRRD: return "ARMISD::VMOVRRD"; - case ARMISD::VMOVDRR: return "ARMISD::VMOVDRR"; - case ARMISD::VMOVhr: return "ARMISD::VMOVhr"; - case ARMISD::VMOVrh: return "ARMISD::VMOVrh"; - case ARMISD::VMOVSR: return "ARMISD::VMOVSR"; - - case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP"; - case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP"; - case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH"; - - case ARMISD::TC_RETURN: return "ARMISD::TC_RETURN"; - - case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER"; - - case ARMISD::DYN_ALLOC: return "ARMISD::DYN_ALLOC"; - - case ARMISD::MEMBARRIER_MCR: return "ARMISD::MEMBARRIER_MCR"; - - case ARMISD::PRELOAD: return "ARMISD::PRELOAD"; - - case ARMISD::LDRD: return "ARMISD::LDRD"; - case ARMISD::STRD: return "ARMISD::STRD"; - - case ARMISD::WIN__CHKSTK: return "ARMISD::WIN__CHKSTK"; - case ARMISD::WIN__DBZCHK: return "ARMISD::WIN__DBZCHK"; - - case ARMISD::PREDICATE_CAST: return "ARMISD::PREDICATE_CAST"; - case ARMISD::VECTOR_REG_CAST: return "ARMISD::VECTOR_REG_CAST"; - case ARMISD::VCMP: return "ARMISD::VCMP"; - case ARMISD::VCMPZ: return "ARMISD::VCMPZ"; - case ARMISD::VTST: return "ARMISD::VTST"; - - case ARMISD::VSHLs: return "ARMISD::VSHLs"; - case ARMISD::VSHLu: return "ARMISD::VSHLu"; - case ARMISD::VSHLIMM: return "ARMISD::VSHLIMM"; - case ARMISD::VSHRsIMM: return "ARMISD::VSHRsIMM"; - case ARMISD::VSHRuIMM: return "ARMISD::VSHRuIMM"; - case ARMISD::VRSHRsIMM: return "ARMISD::VRSHRsIMM"; - case ARMISD::VRSHRuIMM: return "ARMISD::VRSHRuIMM"; - case ARMISD::VRSHRNIMM: return "ARMISD::VRSHRNIMM"; - case ARMISD::VQSHLsIMM: return "ARMISD::VQSHLsIMM"; - case ARMISD::VQSHLuIMM: return "ARMISD::VQSHLuIMM"; - case ARMISD::VQSHLsuIMM: return "ARMISD::VQSHLsuIMM"; - case ARMISD::VQSHRNsIMM: return "ARMISD::VQSHRNsIMM"; - case ARMISD::VQSHRNuIMM: return "ARMISD::VQSHRNuIMM"; - case ARMISD::VQSHRNsuIMM: return "ARMISD::VQSHRNsuIMM"; - case ARMISD::VQRSHRNsIMM: return "ARMISD::VQRSHRNsIMM"; - case ARMISD::VQRSHRNuIMM: return "ARMISD::VQRSHRNuIMM"; - case ARMISD::VQRSHRNsuIMM: return "ARMISD::VQRSHRNsuIMM"; - case ARMISD::VSLIIMM: return "ARMISD::VSLIIMM"; - case ARMISD::VSRIIMM: return "ARMISD::VSRIIMM"; - case ARMISD::VGETLANEu: return "ARMISD::VGETLANEu"; - case ARMISD::VGETLANEs: return "ARMISD::VGETLANEs"; - case ARMISD::VMOVIMM: return "ARMISD::VMOVIMM"; - case ARMISD::VMVNIMM: return "ARMISD::VMVNIMM"; - case ARMISD::VMOVFPIMM: return "ARMISD::VMOVFPIMM"; - case ARMISD::VDUP: return "ARMISD::VDUP"; - case ARMISD::VDUPLANE: return "ARMISD::VDUPLANE"; - case ARMISD::VEXT: return "ARMISD::VEXT"; - case ARMISD::VREV64: return "ARMISD::VREV64"; - case ARMISD::VREV32: return "ARMISD::VREV32"; - case ARMISD::VREV16: return "ARMISD::VREV16"; - case ARMISD::VZIP: return "ARMISD::VZIP"; - case ARMISD::VUZP: return "ARMISD::VUZP"; - case ARMISD::VTRN: return "ARMISD::VTRN"; - case ARMISD::VTBL1: return "ARMISD::VTBL1"; - case ARMISD::VTBL2: return "ARMISD::VTBL2"; - case ARMISD::VMOVN: return "ARMISD::VMOVN"; - case ARMISD::VQMOVNs: return "ARMISD::VQMOVNs"; - case ARMISD::VQMOVNu: return "ARMISD::VQMOVNu"; - case ARMISD::VCVTN: return "ARMISD::VCVTN"; - case ARMISD::VCVTL: return "ARMISD::VCVTL"; - case ARMISD::VMULLs: return "ARMISD::VMULLs"; - case ARMISD::VMULLu: return "ARMISD::VMULLu"; - case ARMISD::VQDMULH: return "ARMISD::VQDMULH"; - case ARMISD::VADDVs: return "ARMISD::VADDVs"; - case ARMISD::VADDVu: return "ARMISD::VADDVu"; - case ARMISD::VADDVps: return "ARMISD::VADDVps"; - case ARMISD::VADDVpu: return "ARMISD::VADDVpu"; - case ARMISD::VADDLVs: return "ARMISD::VADDLVs"; - case ARMISD::VADDLVu: return "ARMISD::VADDLVu"; - case ARMISD::VADDLVAs: return "ARMISD::VADDLVAs"; - case ARMISD::VADDLVAu: return "ARMISD::VADDLVAu"; - case ARMISD::VADDLVps: return "ARMISD::VADDLVps"; - case ARMISD::VADDLVpu: return "ARMISD::VADDLVpu"; - case ARMISD::VADDLVAps: return "ARMISD::VADDLVAps"; - case ARMISD::VADDLVApu: return "ARMISD::VADDLVApu"; - case ARMISD::VMLAVs: return "ARMISD::VMLAVs"; - case ARMISD::VMLAVu: return "ARMISD::VMLAVu"; - case ARMISD::VMLAVps: return "ARMISD::VMLAVps"; - case ARMISD::VMLAVpu: return "ARMISD::VMLAVpu"; - case ARMISD::VMLALVs: return "ARMISD::VMLALVs"; - case ARMISD::VMLALVu: return "ARMISD::VMLALVu"; - case ARMISD::VMLALVps: return "ARMISD::VMLALVps"; - case ARMISD::VMLALVpu: return "ARMISD::VMLALVpu"; - case ARMISD::VMLALVAs: return "ARMISD::VMLALVAs"; - case ARMISD::VMLALVAu: return "ARMISD::VMLALVAu"; - case ARMISD::VMLALVAps: return "ARMISD::VMLALVAps"; - case ARMISD::VMLALVApu: return "ARMISD::VMLALVApu"; - case ARMISD::VMINVu: return "ARMISD::VMINVu"; - case ARMISD::VMINVs: return "ARMISD::VMINVs"; - case ARMISD::VMAXVu: return "ARMISD::VMAXVu"; - case ARMISD::VMAXVs: return "ARMISD::VMAXVs"; - case ARMISD::UMAAL: return "ARMISD::UMAAL"; - case ARMISD::UMLAL: return "ARMISD::UMLAL"; - case ARMISD::SMLAL: return "ARMISD::SMLAL"; - case ARMISD::SMLALBB: return "ARMISD::SMLALBB"; - case ARMISD::SMLALBT: return "ARMISD::SMLALBT"; - case ARMISD::SMLALTB: return "ARMISD::SMLALTB"; - case ARMISD::SMLALTT: return "ARMISD::SMLALTT"; - case ARMISD::SMULWB: return "ARMISD::SMULWB"; - case ARMISD::SMULWT: return "ARMISD::SMULWT"; - case ARMISD::SMLALD: return "ARMISD::SMLALD"; - case ARMISD::SMLALDX: return "ARMISD::SMLALDX"; - case ARMISD::SMLSLD: return "ARMISD::SMLSLD"; - case ARMISD::SMLSLDX: return "ARMISD::SMLSLDX"; - case ARMISD::SMMLAR: return "ARMISD::SMMLAR"; - case ARMISD::SMMLSR: return "ARMISD::SMMLSR"; - case ARMISD::QADD16b: return "ARMISD::QADD16b"; - case ARMISD::QSUB16b: return "ARMISD::QSUB16b"; - case ARMISD::QADD8b: return "ARMISD::QADD8b"; - case ARMISD::QSUB8b: return "ARMISD::QSUB8b"; - case ARMISD::BUILD_VECTOR: return "ARMISD::BUILD_VECTOR"; - case ARMISD::BFI: return "ARMISD::BFI"; - case ARMISD::VORRIMM: return "ARMISD::VORRIMM"; - case ARMISD::VBICIMM: return "ARMISD::VBICIMM"; - case ARMISD::VBSP: return "ARMISD::VBSP"; - case ARMISD::MEMCPY: return "ARMISD::MEMCPY"; - case ARMISD::VLD1DUP: return "ARMISD::VLD1DUP"; - case ARMISD::VLD2DUP: return "ARMISD::VLD2DUP"; - case ARMISD::VLD3DUP: return "ARMISD::VLD3DUP"; - case ARMISD::VLD4DUP: return "ARMISD::VLD4DUP"; - case ARMISD::VLD1_UPD: return "ARMISD::VLD1_UPD"; - case ARMISD::VLD2_UPD: return "ARMISD::VLD2_UPD"; - case ARMISD::VLD3_UPD: return "ARMISD::VLD3_UPD"; - case ARMISD::VLD4_UPD: return "ARMISD::VLD4_UPD"; - case ARMISD::VLD2LN_UPD: return "ARMISD::VLD2LN_UPD"; - case ARMISD::VLD3LN_UPD: return "ARMISD::VLD3LN_UPD"; - case ARMISD::VLD4LN_UPD: return "ARMISD::VLD4LN_UPD"; - case ARMISD::VLD1DUP_UPD: return "ARMISD::VLD1DUP_UPD"; - case ARMISD::VLD2DUP_UPD: return "ARMISD::VLD2DUP_UPD"; - case ARMISD::VLD3DUP_UPD: return "ARMISD::VLD3DUP_UPD"; - case ARMISD::VLD4DUP_UPD: return "ARMISD::VLD4DUP_UPD"; - case ARMISD::VST1_UPD: return "ARMISD::VST1_UPD"; - case ARMISD::VST2_UPD: return "ARMISD::VST2_UPD"; - case ARMISD::VST3_UPD: return "ARMISD::VST3_UPD"; - case ARMISD::VST4_UPD: return "ARMISD::VST4_UPD"; - case ARMISD::VST2LN_UPD: return "ARMISD::VST2LN_UPD"; - case ARMISD::VST3LN_UPD: return "ARMISD::VST3LN_UPD"; - case ARMISD::VST4LN_UPD: return "ARMISD::VST4LN_UPD"; - case ARMISD::WLS: return "ARMISD::WLS"; - case ARMISD::LE: return "ARMISD::LE"; - case ARMISD::LOOP_DEC: return "ARMISD::LOOP_DEC"; - case ARMISD::CSINV: return "ARMISD::CSINV"; - case ARMISD::CSNEG: return "ARMISD::CSNEG"; - case ARMISD::CSINC: return "ARMISD::CSINC"; + case ARMISD::FIRST_NUMBER: + break; + MAKE_CASE(ARMISD::Wrapper) + MAKE_CASE(ARMISD::WrapperPIC) + MAKE_CASE(ARMISD::WrapperJT) + MAKE_CASE(ARMISD::COPY_STRUCT_BYVAL) + MAKE_CASE(ARMISD::CALL) + MAKE_CASE(ARMISD::CALL_PRED) + MAKE_CASE(ARMISD::CALL_NOLINK) + MAKE_CASE(ARMISD::tSECALL) + MAKE_CASE(ARMISD::BRCOND) + MAKE_CASE(ARMISD::BR_JT) + MAKE_CASE(ARMISD::BR2_JT) + MAKE_CASE(ARMISD::RET_FLAG) + MAKE_CASE(ARMISD::SERET_FLAG) + MAKE_CASE(ARMISD::INTRET_FLAG) + MAKE_CASE(ARMISD::PIC_ADD) + MAKE_CASE(ARMISD::CMP) + MAKE_CASE(ARMISD::CMN) + MAKE_CASE(ARMISD::CMPZ) + MAKE_CASE(ARMISD::CMPFP) + MAKE_CASE(ARMISD::CMPFPE) + MAKE_CASE(ARMISD::CMPFPw0) + MAKE_CASE(ARMISD::CMPFPEw0) + MAKE_CASE(ARMISD::BCC_i64) + MAKE_CASE(ARMISD::FMSTAT) + MAKE_CASE(ARMISD::CMOV) + MAKE_CASE(ARMISD::SUBS) + MAKE_CASE(ARMISD::SSAT) + MAKE_CASE(ARMISD::USAT) + MAKE_CASE(ARMISD::ASRL) + MAKE_CASE(ARMISD::LSRL) + MAKE_CASE(ARMISD::LSLL) + MAKE_CASE(ARMISD::SRL_FLAG) + MAKE_CASE(ARMISD::SRA_FLAG) + MAKE_CASE(ARMISD::RRX) + MAKE_CASE(ARMISD::ADDC) + MAKE_CASE(ARMISD::ADDE) + MAKE_CASE(ARMISD::SUBC) + MAKE_CASE(ARMISD::SUBE) + MAKE_CASE(ARMISD::LSLS) + MAKE_CASE(ARMISD::VMOVRRD) + MAKE_CASE(ARMISD::VMOVDRR) + MAKE_CASE(ARMISD::VMOVhr) + MAKE_CASE(ARMISD::VMOVrh) + MAKE_CASE(ARMISD::VMOVSR) + MAKE_CASE(ARMISD::EH_SJLJ_SETJMP) + MAKE_CASE(ARMISD::EH_SJLJ_LONGJMP) + MAKE_CASE(ARMISD::EH_SJLJ_SETUP_DISPATCH) + MAKE_CASE(ARMISD::TC_RETURN) + MAKE_CASE(ARMISD::THREAD_POINTER) + MAKE_CASE(ARMISD::DYN_ALLOC) + MAKE_CASE(ARMISD::MEMBARRIER_MCR) + MAKE_CASE(ARMISD::PRELOAD) + MAKE_CASE(ARMISD::LDRD) + MAKE_CASE(ARMISD::STRD) + MAKE_CASE(ARMISD::WIN__CHKSTK) + MAKE_CASE(ARMISD::WIN__DBZCHK) + MAKE_CASE(ARMISD::PREDICATE_CAST) + MAKE_CASE(ARMISD::VECTOR_REG_CAST) + MAKE_CASE(ARMISD::MVESEXT) + MAKE_CASE(ARMISD::MVEZEXT) + MAKE_CASE(ARMISD::MVETRUNC) + MAKE_CASE(ARMISD::VCMP) + MAKE_CASE(ARMISD::VCMPZ) + MAKE_CASE(ARMISD::VTST) + MAKE_CASE(ARMISD::VSHLs) + MAKE_CASE(ARMISD::VSHLu) + MAKE_CASE(ARMISD::VSHLIMM) + MAKE_CASE(ARMISD::VSHRsIMM) + MAKE_CASE(ARMISD::VSHRuIMM) + MAKE_CASE(ARMISD::VRSHRsIMM) + MAKE_CASE(ARMISD::VRSHRuIMM) + MAKE_CASE(ARMISD::VRSHRNIMM) + MAKE_CASE(ARMISD::VQSHLsIMM) + MAKE_CASE(ARMISD::VQSHLuIMM) + MAKE_CASE(ARMISD::VQSHLsuIMM) + MAKE_CASE(ARMISD::VQSHRNsIMM) + MAKE_CASE(ARMISD::VQSHRNuIMM) + MAKE_CASE(ARMISD::VQSHRNsuIMM) + MAKE_CASE(ARMISD::VQRSHRNsIMM) + MAKE_CASE(ARMISD::VQRSHRNuIMM) + MAKE_CASE(ARMISD::VQRSHRNsuIMM) + MAKE_CASE(ARMISD::VSLIIMM) + MAKE_CASE(ARMISD::VSRIIMM) + MAKE_CASE(ARMISD::VGETLANEu) + MAKE_CASE(ARMISD::VGETLANEs) + MAKE_CASE(ARMISD::VMOVIMM) + MAKE_CASE(ARMISD::VMVNIMM) + MAKE_CASE(ARMISD::VMOVFPIMM) + MAKE_CASE(ARMISD::VDUP) + MAKE_CASE(ARMISD::VDUPLANE) + MAKE_CASE(ARMISD::VEXT) + MAKE_CASE(ARMISD::VREV64) + MAKE_CASE(ARMISD::VREV32) + MAKE_CASE(ARMISD::VREV16) + MAKE_CASE(ARMISD::VZIP) + MAKE_CASE(ARMISD::VUZP) + MAKE_CASE(ARMISD::VTRN) + MAKE_CASE(ARMISD::VTBL1) + MAKE_CASE(ARMISD::VTBL2) + MAKE_CASE(ARMISD::VMOVN) + MAKE_CASE(ARMISD::VQMOVNs) + MAKE_CASE(ARMISD::VQMOVNu) + MAKE_CASE(ARMISD::VCVTN) + MAKE_CASE(ARMISD::VCVTL) + MAKE_CASE(ARMISD::VIDUP) + MAKE_CASE(ARMISD::VMULLs) + MAKE_CASE(ARMISD::VMULLu) + MAKE_CASE(ARMISD::VQDMULH) + MAKE_CASE(ARMISD::VADDVs) + MAKE_CASE(ARMISD::VADDVu) + MAKE_CASE(ARMISD::VADDVps) + MAKE_CASE(ARMISD::VADDVpu) + MAKE_CASE(ARMISD::VADDLVs) + MAKE_CASE(ARMISD::VADDLVu) + MAKE_CASE(ARMISD::VADDLVAs) + MAKE_CASE(ARMISD::VADDLVAu) + MAKE_CASE(ARMISD::VADDLVps) + MAKE_CASE(ARMISD::VADDLVpu) + MAKE_CASE(ARMISD::VADDLVAps) + MAKE_CASE(ARMISD::VADDLVApu) + MAKE_CASE(ARMISD::VMLAVs) + MAKE_CASE(ARMISD::VMLAVu) + MAKE_CASE(ARMISD::VMLAVps) + MAKE_CASE(ARMISD::VMLAVpu) + MAKE_CASE(ARMISD::VMLALVs) + MAKE_CASE(ARMISD::VMLALVu) + MAKE_CASE(ARMISD::VMLALVps) + MAKE_CASE(ARMISD::VMLALVpu) + MAKE_CASE(ARMISD::VMLALVAs) + MAKE_CASE(ARMISD::VMLALVAu) + MAKE_CASE(ARMISD::VMLALVAps) + MAKE_CASE(ARMISD::VMLALVApu) + MAKE_CASE(ARMISD::VMINVu) + MAKE_CASE(ARMISD::VMINVs) + MAKE_CASE(ARMISD::VMAXVu) + MAKE_CASE(ARMISD::VMAXVs) + MAKE_CASE(ARMISD::UMAAL) + MAKE_CASE(ARMISD::UMLAL) + MAKE_CASE(ARMISD::SMLAL) + MAKE_CASE(ARMISD::SMLALBB) + MAKE_CASE(ARMISD::SMLALBT) + MAKE_CASE(ARMISD::SMLALTB) + MAKE_CASE(ARMISD::SMLALTT) + MAKE_CASE(ARMISD::SMULWB) + MAKE_CASE(ARMISD::SMULWT) + MAKE_CASE(ARMISD::SMLALD) + MAKE_CASE(ARMISD::SMLALDX) + MAKE_CASE(ARMISD::SMLSLD) + MAKE_CASE(ARMISD::SMLSLDX) + MAKE_CASE(ARMISD::SMMLAR) + MAKE_CASE(ARMISD::SMMLSR) + MAKE_CASE(ARMISD::QADD16b) + MAKE_CASE(ARMISD::QSUB16b) + MAKE_CASE(ARMISD::QADD8b) + MAKE_CASE(ARMISD::QSUB8b) + MAKE_CASE(ARMISD::UQADD16b) + MAKE_CASE(ARMISD::UQSUB16b) + MAKE_CASE(ARMISD::UQADD8b) + MAKE_CASE(ARMISD::UQSUB8b) + MAKE_CASE(ARMISD::BUILD_VECTOR) + MAKE_CASE(ARMISD::BFI) + MAKE_CASE(ARMISD::VORRIMM) + MAKE_CASE(ARMISD::VBICIMM) + MAKE_CASE(ARMISD::VBSP) + MAKE_CASE(ARMISD::MEMCPY) + MAKE_CASE(ARMISD::VLD1DUP) + MAKE_CASE(ARMISD::VLD2DUP) + MAKE_CASE(ARMISD::VLD3DUP) + MAKE_CASE(ARMISD::VLD4DUP) + MAKE_CASE(ARMISD::VLD1_UPD) + MAKE_CASE(ARMISD::VLD2_UPD) + MAKE_CASE(ARMISD::VLD3_UPD) + MAKE_CASE(ARMISD::VLD4_UPD) + MAKE_CASE(ARMISD::VLD1x2_UPD) + MAKE_CASE(ARMISD::VLD1x3_UPD) + MAKE_CASE(ARMISD::VLD1x4_UPD) + MAKE_CASE(ARMISD::VLD2LN_UPD) + MAKE_CASE(ARMISD::VLD3LN_UPD) + MAKE_CASE(ARMISD::VLD4LN_UPD) + MAKE_CASE(ARMISD::VLD1DUP_UPD) + MAKE_CASE(ARMISD::VLD2DUP_UPD) + MAKE_CASE(ARMISD::VLD3DUP_UPD) + MAKE_CASE(ARMISD::VLD4DUP_UPD) + MAKE_CASE(ARMISD::VST1_UPD) + MAKE_CASE(ARMISD::VST2_UPD) + MAKE_CASE(ARMISD::VST3_UPD) + MAKE_CASE(ARMISD::VST4_UPD) + MAKE_CASE(ARMISD::VST1x2_UPD) + MAKE_CASE(ARMISD::VST1x3_UPD) + MAKE_CASE(ARMISD::VST1x4_UPD) + MAKE_CASE(ARMISD::VST2LN_UPD) + MAKE_CASE(ARMISD::VST3LN_UPD) + MAKE_CASE(ARMISD::VST4LN_UPD) + MAKE_CASE(ARMISD::WLS) + MAKE_CASE(ARMISD::WLSSETUP) + MAKE_CASE(ARMISD::LE) + MAKE_CASE(ARMISD::LOOP_DEC) + MAKE_CASE(ARMISD::CSINV) + MAKE_CASE(ARMISD::CSNEG) + MAKE_CASE(ARMISD::CSINC) + MAKE_CASE(ARMISD::MEMCPYLOOP) + MAKE_CASE(ARMISD::MEMSETLOOP) +#undef MAKE_CASE } return nullptr; } @@ -1817,8 +1835,9 @@ EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, return getPointerTy(DL); // MVE has a predicate register. - if (Subtarget->hasMVEIntegerOps() && - (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8)) + if ((Subtarget->hasMVEIntegerOps() && + (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8)) || + (Subtarget->hasMVEFloatOps() && (VT == MVT::v4f32 || VT == MVT::v8f16))) return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount()); return VT.changeVectorElementTypeToInteger(); } @@ -1998,8 +2017,10 @@ ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC, return CallingConv::PreserveMost; case CallingConv::ARM_AAPCS_VFP: case CallingConv::Swift: + case CallingConv::SwiftTail: return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP; case CallingConv::C: + case CallingConv::Tail: if (!Subtarget->isAAPCS_ABI()) return CallingConv::ARM_APCS; else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && @@ -2176,19 +2197,31 @@ SDValue ARMTargetLowering::LowerCallResult( return Chain; } -/// LowerMemOpCallTo - Store the argument to the stack. -SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, - SDValue Arg, const SDLoc &dl, - SelectionDAG &DAG, - const CCValAssign &VA, - ISD::ArgFlagsTy Flags) const { - unsigned LocMemOffset = VA.getLocMemOffset(); - SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); - PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), - StackPtr, PtrOff); - return DAG.getStore( - Chain, dl, Arg, PtrOff, - MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset)); +std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg( + const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr, + bool IsTailCall, int SPDiff) const { + SDValue DstAddr; + MachinePointerInfo DstInfo; + int32_t Offset = VA.getLocMemOffset(); + MachineFunction &MF = DAG.getMachineFunction(); + + if (IsTailCall) { + Offset += SPDiff; + auto PtrVT = getPointerTy(DAG.getDataLayout()); + int Size = VA.getLocVT().getFixedSizeInBits() / 8; + int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true); + DstAddr = DAG.getFrameIndex(FI, PtrVT); + DstInfo = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); + } else { + SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl); + DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), + StackPtr, PtrOff); + DstInfo = + MachinePointerInfo::getStack(DAG.getMachineFunction(), Offset); + } + + return std::make_pair(DstAddr, DstInfo); } void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, @@ -2197,7 +2230,8 @@ void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, CCValAssign &VA, CCValAssign &NextVA, SDValue &StackPtr, SmallVectorImpl<SDValue> &MemOpChains, - ISD::ArgFlagsTy Flags) const { + bool IsTailCall, + int SPDiff) const { SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32), Arg); unsigned id = Subtarget->isLittle() ? 0 : 1; @@ -2211,12 +2245,20 @@ void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout())); - MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id), - dl, DAG, NextVA, - Flags)); + SDValue DstAddr; + MachinePointerInfo DstInfo; + std::tie(DstAddr, DstInfo) = + computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff); + MemOpChains.push_back( + DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo)); } } +static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) { + return (CC == CallingConv::Fast && GuaranteeTailCalls) || + CC == CallingConv::Tail || CC == CallingConv::SwiftTail; +} + /// LowerCall - Lowering a call into a callseq_start <- /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter /// nodes. @@ -2241,6 +2283,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet(); bool isThisReturn = false; bool isCmseNSCall = false; + bool isSibCall = false; bool PreferIndirect = false; // Determine whether this is a non-secure function call. @@ -2277,15 +2320,20 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, Callee, CallConv, isVarArg, isStructRet, MF.getFunction().hasStructRetAttr(), Outs, OutVals, Ins, DAG, PreferIndirect); - if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall()) - report_fatal_error("failed to perform tail call elimination on a call " - "site marked musttail"); + + if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt && + CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail) + isSibCall = true; + // We don't support GuaranteedTailCallOpt for ARM, only automatically // detected sibcalls. if (isTailCall) ++NumTailCalls; } + if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall()) + report_fatal_error("failed to perform tail call elimination on a call " + "site marked musttail"); // Analyze operands of the call, assigning locations to each operand. SmallVector<CCValAssign, 16> ArgLocs; CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, @@ -2295,13 +2343,40 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // Get a count of how many bytes are to be pushed on the stack. unsigned NumBytes = CCInfo.getNextStackOffset(); - if (isTailCall) { - // For tail calls, memory operands are available in our caller's stack. + // SPDiff is the byte offset of the call's argument area from the callee's. + // Stores to callee stack arguments will be placed in FixedStackSlots offset + // by this amount for a tail call. In a sibling call it must be 0 because the + // caller will deallocate the entire stack and the callee still expects its + // arguments to begin at SP+0. Completely unused for non-tail calls. + int SPDiff = 0; + + if (isTailCall && !isSibCall) { + auto FuncInfo = MF.getInfo<ARMFunctionInfo>(); + unsigned NumReusableBytes = FuncInfo->getArgumentStackSize(); + + // Since callee will pop argument stack as a tail call, we must keep the + // popped size 16-byte aligned. + Align StackAlign = DAG.getDataLayout().getStackAlignment(); + NumBytes = alignTo(NumBytes, StackAlign); + + // SPDiff will be negative if this tail call requires more space than we + // would automatically have in our incoming argument space. Positive if we + // can actually shrink the stack. + SPDiff = NumReusableBytes - NumBytes; + + // If this call requires more stack than we have available from + // LowerFormalArguments, tell FrameLowering to reserve space for it. + if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff) + AFI->setArgRegsSaveSize(-SPDiff); + } + + if (isSibCall) { + // For sibling tail calls, memory operands are available in our caller's stack. NumBytes = 0; } else { // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass - Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); + Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl); } SDValue StackPtr = @@ -2310,6 +2385,13 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, RegsToPassVector RegsToPass; SmallVector<SDValue, 8> MemOpChains; + // During a tail call, stores to the argument area must happen after all of + // the function's incoming arguments have been loaded because they may alias. + // This is done by folding in a TokenFactor from LowerFormalArguments, but + // there's no point in doing so repeatedly so this tracks whether that's + // happened yet. + bool AfterFormalArgLoads = false; + // Walk the register/memloc assignments, inserting copies/loads. In the case // of tail call optimization, arguments are handled later. for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); @@ -2338,6 +2420,11 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, break; } + if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) { + Chain = DAG.getStackArgumentTokenFactor(Chain); + AfterFormalArgLoads = true; + } + // f16 arguments have their size extended to 4 bytes and passed as if they // had been copied to the LSBs of a 32-bit register. // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI) @@ -2367,21 +2454,23 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, DAG.getConstant(1, dl, MVT::i32)); PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i], - StackPtr, MemOpChains, Flags); + StackPtr, MemOpChains, isTailCall, SPDiff); VA = ArgLocs[++i]; // skip ahead to next loc if (VA.isRegLoc()) { PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i], - StackPtr, MemOpChains, Flags); + StackPtr, MemOpChains, isTailCall, SPDiff); } else { assert(VA.isMemLoc()); - - MemOpChains.push_back( - LowerMemOpCallTo(Chain, StackPtr, Op1, dl, DAG, VA, Flags)); + SDValue DstAddr; + MachinePointerInfo DstInfo; + std::tie(DstAddr, DstInfo) = + computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff); + MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo)); } } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) { PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i], - StackPtr, MemOpChains, Flags); + StackPtr, MemOpChains, isTailCall, SPDiff); } else if (VA.isRegLoc()) { if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() && Outs[0].VT == MVT::i32) { @@ -2431,9 +2520,10 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (Flags.getByValSize() > 4*offset) { auto PtrVT = getPointerTy(DAG.getDataLayout()); - unsigned LocMemOffset = VA.getLocMemOffset(); - SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); - SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff); + SDValue Dst; + MachinePointerInfo DstInfo; + std::tie(Dst, DstInfo) = + computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff); SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl); SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset); SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl, @@ -2446,11 +2536,15 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, Ops)); } - } else if (!isTailCall) { + } else { assert(VA.isMemLoc()); + SDValue DstAddr; + MachinePointerInfo DstInfo; + std::tie(DstAddr, DstInfo) = + computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff); - MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, - dl, DAG, VA, Flags)); + SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo); + MemOpChains.push_back(Store); } } @@ -2614,10 +2708,24 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL; } + // We don't usually want to end the call-sequence here because we would tidy + // the frame up *after* the call, however in the ABI-changing tail-call case + // we've carefully laid out the parameters so that when sp is reset they'll be + // in the correct location. + if (isTailCall && !isSibCall) { + Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), + DAG.getIntPtrConstant(0, dl, true), InFlag, dl); + InFlag = Chain.getValue(1); + } + std::vector<SDValue> Ops; Ops.push_back(Chain); Ops.push_back(Callee); + if (isTailCall) { + Ops.push_back(DAG.getTargetConstant(SPDiff, dl, MVT::i32)); + } + // Add argument registers to the end of the list so that they are known live // into the call. for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) @@ -2662,8 +2770,16 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, InFlag = Chain.getValue(1); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); + // If we're guaranteeing tail-calls will be honoured, the callee must + // pop its own argument stack on return. But this call is *not* a tail call so + // we need to undo that after it returns to restore the status-quo. + bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; + uint64_t CalleePopBytes = + canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1ULL; + Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), - DAG.getIntPtrConstant(0, dl, true), InFlag, dl); + DAG.getIntPtrConstant(CalleePopBytes, dl, true), + InFlag, dl); if (!Ins.empty()) InFlag = Chain.getValue(1); @@ -2804,6 +2920,9 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization( if (CallerF.hasFnAttribute("interrupt")) return false; + if (canGuaranteeTCO(CalleeCC, getTargetMachine().Options.GuaranteedTailCallOpt)) + return CalleeCC == CallerCC; + // Also avoid sibcall optimization if either caller or callee uses struct // return semantics. if (isCalleeStructRet || isCallerStructRet) @@ -3550,9 +3669,7 @@ ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { /// Return true if all users of V are within function F, looking through /// ConstantExprs. static bool allUsersAreInFunction(const Value *V, const Function *F) { - SmallVector<const User*,4> Worklist; - for (auto *U : V->users()) - Worklist.push_back(U); + SmallVector<const User*,4> Worklist(V->users()); while (!Worklist.empty()) { auto *U = Worklist.pop_back_val(); if (isa<ConstantExpr>(U)) { @@ -3599,7 +3716,7 @@ static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, // from .data to .text. This is not allowed in position-independent code. auto *Init = GVar->getInitializer(); if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) && - Init->needsRelocation()) + Init->needsDynamicRelocation()) return SDValue(); // The constant islands pass can only really deal with alignment requests @@ -4454,7 +4571,17 @@ SDValue ARMTargetLowering::LowerFormalArguments( } } - AFI->setArgumentStackSize(CCInfo.getNextStackOffset()); + unsigned StackArgSize = CCInfo.getNextStackOffset(); + bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; + if (canGuaranteeTCO(CallConv, TailCallOpt)) { + // The only way to guarantee a tail call is if the callee restores its + // argument area, but it must also keep the stack aligned when doing so. + const DataLayout &DL = DAG.getDataLayout(); + StackArgSize = alignTo(StackArgSize, DL.getStackAlignment()); + + AFI->setArgumentStackToRestore(StackArgSize); + } + AFI->setArgumentStackSize(StackArgSize); if (CCInfo.getNextStackOffset() > 0 && AFI->isCmseNSEntryFunction()) { DiagnosticInfoUnsupported Diag( @@ -4826,8 +4953,8 @@ SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op, return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); } -static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG, - const ARMSubtarget *Subtarget) { +static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, + const ARMSubtarget *Subtarget) { EVT VT = Op.getValueType(); if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) return SDValue(); @@ -4835,15 +4962,40 @@ static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG, return SDValue(); unsigned NewOpcode; - bool IsAdd = Op->getOpcode() == ISD::SADDSAT; switch (VT.getSimpleVT().SimpleTy) { default: return SDValue(); case MVT::i8: - NewOpcode = IsAdd ? ARMISD::QADD8b : ARMISD::QSUB8b; + switch (Op->getOpcode()) { + case ISD::UADDSAT: + NewOpcode = ARMISD::UQADD8b; + break; + case ISD::SADDSAT: + NewOpcode = ARMISD::QADD8b; + break; + case ISD::USUBSAT: + NewOpcode = ARMISD::UQSUB8b; + break; + case ISD::SSUBSAT: + NewOpcode = ARMISD::QSUB8b; + break; + } break; case MVT::i16: - NewOpcode = IsAdd ? ARMISD::QADD16b : ARMISD::QSUB16b; + switch (Op->getOpcode()) { + case ISD::UADDSAT: + NewOpcode = ARMISD::UQADD16b; + break; + case ISD::SADDSAT: + NewOpcode = ARMISD::QADD16b; + break; + case ISD::USUBSAT: + NewOpcode = ARMISD::UQSUB16b; + break; + case ISD::SSUBSAT: + NewOpcode = ARMISD::QSUB16b; + break; + } break; } @@ -5224,8 +5376,6 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { std::swap(TVal, FVal); CC = ISD::getSetCCInverse(CC, LHS.getValueType()); } - if (TVal == 0) - TrueVal = DAG.getRegister(ARM::ZR, MVT::i32); // Drops F's value because we can get it by inverting/negating TVal. FalseVal = TrueVal; @@ -6117,6 +6267,48 @@ SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op, return DAG.getMergeValues({And, Chain}, dl); } +SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op->getOperand(0); + SDValue RMValue = Op->getOperand(1); + + // The rounding mode is in bits 23:22 of the FPSCR. + // The llvm.set.rounding argument value to ARM rounding mode value mapping + // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is + // ((arg - 1) & 3) << 22). + // + // It is expected that the argument of llvm.set.rounding is within the + // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is + // responsibility of the code generated llvm.set.rounding to ensure this + // condition. + + // Calculate new value of FPSCR[23:22]. + RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue, + DAG.getConstant(1, DL, MVT::i32)); + RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue, + DAG.getConstant(0x3, DL, MVT::i32)); + RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue, + DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32)); + + // Get current value of FPSCR. + SDValue Ops[] = {Chain, + DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)}; + SDValue FPSCR = + DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); + Chain = FPSCR.getValue(1); + FPSCR = FPSCR.getValue(0); + + // Put new rounding mode into FPSCR[23:22]. + const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos); + FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR, + DAG.getConstant(RMMask, DL, MVT::i32)); + FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue); + SDValue Ops2[] = { + Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR}; + return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); +} + static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) { SDLoc dl(N); @@ -6706,12 +6898,10 @@ static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, return SDValue(); // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff. uint64_t BitMask = 0xff; - uint64_t Val = 0; unsigned ImmMask = 1; Imm = 0; for (int ByteNum = 0; ByteNum < 8; ++ByteNum) { if (((SplatBits | SplatUndef) & BitMask) == BitMask) { - Val |= BitMask; Imm |= ImmMask; } else if ((SplatBits & BitMask) != 0) { return SDValue(); @@ -6923,35 +7113,6 @@ static bool isVEXTMask(ArrayRef<int> M, EVT VT, return true; } -/// isVREVMask - Check if a vector shuffle corresponds to a VREV -/// instruction with the specified blocksize. (The order of the elements -/// within each block of the vector is reversed.) -static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { - assert((BlockSize==16 || BlockSize==32 || BlockSize==64) && - "Only possible block sizes for VREV are: 16, 32, 64"); - - unsigned EltSz = VT.getScalarSizeInBits(); - if (EltSz == 64) - return false; - - unsigned NumElts = VT.getVectorNumElements(); - unsigned BlockElts = M[0] + 1; - // If the first shuffle index is UNDEF, be optimistic. - if (M[0] < 0) - BlockElts = BlockSize / EltSz; - - if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz) - return false; - - for (unsigned i = 0; i < NumElts; ++i) { - if (M[i] < 0) continue; // ignore UNDEF indices - if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts)) - return false; - } - - return true; -} - static bool isVTBLMask(ArrayRef<int> M, EVT VT) { // We can handle <8 x i8> vector shuffles. If the index in the mask is out of // range, then 0 is placed into the resulting vector. So pretty much any mask @@ -7224,11 +7385,11 @@ static bool isReverseMask(ArrayRef<int> M, EVT VT) { return true; } -static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top) { +static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) { unsigned NumElts = VT.getVectorNumElements(); // Make sure the mask has the right size. if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8)) - return false; + return false; // If Top // Look for <0, N, 2, N+2, 4, N+4, ..>. @@ -7237,10 +7398,33 @@ static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top) { // Look for <0, N+1, 2, N+3, 4, N+5, ..> // This inserts Input1 into Input2 unsigned Offset = Top ? 0 : 1; - for (unsigned i = 0; i < NumElts; i+=2) { + unsigned N = SingleSource ? 0 : NumElts; + for (unsigned i = 0; i < NumElts; i += 2) { if (M[i] >= 0 && M[i] != (int)i) return false; - if (M[i+1] >= 0 && M[i+1] != (int)(NumElts + i + Offset)) + if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset)) + return false; + } + + return true; +} + +static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) { + unsigned NumElts = ToVT.getVectorNumElements(); + if (NumElts != M.size()) + return false; + + // Test if the Trunc can be convertable to a VMOVN with this shuffle. We are + // looking for patterns of: + // !rev: 0 N/2 1 N/2+1 2 N/2+2 ... + // rev: N/2 0 N/2+1 1 N/2+2 2 ... + + unsigned Off0 = rev ? NumElts / 2 : 0; + unsigned Off1 = rev ? 0 : NumElts / 2; + for (unsigned i = 0; i < NumElts; i += 2) { + if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2)) + return false; + if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2)) return false; } @@ -7425,6 +7609,39 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, return Base; } +static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, + const ARMSubtarget *ST) { + if (!ST->hasMVEIntegerOps()) + return SDValue(); + + // We are looking for a buildvector where each element is Op[0] + i*N + EVT VT = Op.getValueType(); + SDValue Op0 = Op.getOperand(0); + unsigned NumElts = VT.getVectorNumElements(); + + // Get the increment value from operand 1 + SDValue Op1 = Op.getOperand(1); + if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 || + !isa<ConstantSDNode>(Op1.getOperand(1))) + return SDValue(); + unsigned N = Op1.getConstantOperandVal(1); + if (N != 1 && N != 2 && N != 4 && N != 8) + return SDValue(); + + // Check that each other operand matches + for (unsigned I = 2; I < NumElts; I++) { + SDValue OpI = Op.getOperand(I); + if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 || + !isa<ConstantSDNode>(OpI.getOperand(1)) || + OpI.getConstantOperandVal(1) != I * N) + return SDValue(); + } + + SDLoc DL(Op); + return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0, + DAG.getConstant(N, DL, MVT::i32)); +} + // If this is a case we can't handle, return null and let the default // expansion code take care of it. SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, @@ -7436,6 +7653,9 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) return LowerBUILD_VECTOR_i1(Op, DAG, ST); + if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST)) + return R; + APInt SplatBits, SplatUndef; unsigned SplatBitSize; bool HasAnyUndefs; @@ -7474,6 +7694,18 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val); } } + + // If we are under MVE, generate a VDUP(constant), bitcast to the original + // type. + if (ST->hasMVEIntegerOps() && + (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) { + EVT DupVT = SplatBitSize == 32 ? MVT::v4i32 + : SplatBitSize == 16 ? MVT::v8i16 + : MVT::v16i8; + SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32); + SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const); + return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup); + } } } @@ -7947,7 +8179,8 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { isReverseMask(M, VT)) return true; else if (Subtarget->hasMVEIntegerOps() && - (isVMOVNMask(M, VT, 0) || isVMOVNMask(M, VT, 1))) + (isVMOVNMask(M, VT, true, false) || + isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true))) return true; else return false; @@ -7981,7 +8214,8 @@ static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, VT.getVectorElementType() == MVT::f32) return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS); // vrev <4 x i16> -> VREV32 - if (VT.getVectorElementType() == MVT::i16) + if (VT.getVectorElementType() == MVT::i16 || + VT.getVectorElementType() == MVT::f16) return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS); // vrev <4 x i8> -> VREV16 assert(VT.getVectorElementType() == MVT::i8); @@ -8192,8 +8426,8 @@ static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, Input = Op->getOperand(1); Elt -= 4; } - SDValue BitCast = DAG.getBitcast(MVT::v4i32, Input); - Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, BitCast, + SDValue BitCast = DAG.getBitcast(MVT::v4f32, Input); + Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast, DAG.getConstant(Elt, dl, MVT::i32)); } } @@ -8212,19 +8446,70 @@ static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]); SDValue NewShuffle = DAG.getVectorShuffle( VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask); - SDValue BitCast = DAG.getBitcast(MVT::v4i32, NewShuffle); + SDValue BitCast = DAG.getBitcast(MVT::v4f32, NewShuffle); for (int Part = 0; Part < 4; ++Part) if (!Parts[Part]) - Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, + Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast, DAG.getConstant(Part, dl, MVT::i32)); } // Build a vector out of the various parts and bitcast it back to the original // type. - SDValue NewVec = DAG.getBuildVector(MVT::v4i32, dl, Parts); + SDValue NewVec = DAG.getNode(ARMISD::BUILD_VECTOR, dl, MVT::v4f32, Parts); return DAG.getBitcast(VT, NewVec); } +static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, + ArrayRef<int> ShuffleMask, + SelectionDAG &DAG) { + SDValue V1 = Op.getOperand(0); + SDValue V2 = Op.getOperand(1); + EVT VT = Op.getValueType(); + unsigned NumElts = VT.getVectorNumElements(); + + // An One-Off Identity mask is one that is mostly an identity mask from as + // single source but contains a single element out-of-place, either from a + // different vector or from another position in the same vector. As opposed to + // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert + // pair directly. + auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset, + int &OffElement) { + OffElement = -1; + int NonUndef = 0; + for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) { + if (Mask[i] == -1) + continue; + NonUndef++; + if (Mask[i] != i + BaseOffset) { + if (OffElement == -1) + OffElement = i; + else + return false; + } + } + return NonUndef > 2 && OffElement != -1; + }; + int OffElement; + SDValue VInput; + if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement)) + VInput = V1; + else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement)) + VInput = V2; + else + return SDValue(); + + SDLoc dl(Op); + EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16 + ? MVT::i32 + : VT.getScalarType(); + SDValue Elt = DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, SVT, + ShuffleMask[OffElement] < (int)NumElts ? V1 : V2, + DAG.getVectorIdxConstant(ShuffleMask[OffElement] % NumElts, dl)); + return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt, + DAG.getVectorIdxConstant(OffElement % NumElts, dl)); +} + static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST) { SDValue V1 = Op.getOperand(0); @@ -8311,12 +8596,15 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, } } if (ST->hasMVEIntegerOps()) { - if (isVMOVNMask(ShuffleMask, VT, 0)) + if (isVMOVNMask(ShuffleMask, VT, false, false)) return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1, DAG.getConstant(0, dl, MVT::i32)); - if (isVMOVNMask(ShuffleMask, VT, 1)) + if (isVMOVNMask(ShuffleMask, VT, true, false)) return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2, DAG.getConstant(1, dl, MVT::i32)); + if (isVMOVNMask(ShuffleMask, VT, true, true)) + return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1, + DAG.getConstant(1, dl, MVT::i32)); } // Also check for these shuffles through CONCAT_VECTORS: we canonicalize @@ -8358,6 +8646,10 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, } } + if (ST->hasMVEIntegerOps() && EltSize <= 32) + if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG)) + return V; + // If the shuffle is not directly supported and it has 4 elements, use // the PerfectShuffle-generated table to synthesize it from other shuffles. unsigned NumElts = VT.getVectorNumElements(); @@ -8643,13 +8935,13 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, } // Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0). -static SDValue LowerTruncatei1(SDValue N, SelectionDAG &DAG, +static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) { assert(ST->hasMVEIntegerOps() && "Expected MVE!"); - EVT VT = N.getValueType(); + EVT VT = N->getValueType(0); assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) && "Expected a vector i1 type!"); - SDValue Op = N.getOperand(0); + SDValue Op = N->getOperand(0); EVT FromVT = Op.getValueType(); SDLoc DL(N); @@ -8659,6 +8951,99 @@ static SDValue LowerTruncatei1(SDValue N, SelectionDAG &DAG, DAG.getCondCode(ISD::SETNE)); } +static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, + const ARMSubtarget *Subtarget) { + if (!Subtarget->hasMVEIntegerOps()) + return SDValue(); + + EVT ToVT = N->getValueType(0); + if (ToVT.getScalarType() == MVT::i1) + return LowerTruncatei1(N, DAG, Subtarget); + + // MVE does not have a single instruction to perform the truncation of a v4i32 + // into the lower half of a v8i16, in the same way that a NEON vmovn would. + // Most of the instructions in MVE follow the 'Beats' system, where moving + // values from different lanes is usually something that the instructions + // avoid. + // + // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B, + // which take a the top/bottom half of a larger lane and extend it (or do the + // opposite, truncating into the top/bottom lane from a larger lane). Note + // that because of the way we widen lanes, a v4i16 is really a v4i32 using the + // bottom 16bits from each vector lane. This works really well with T/B + // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need + // to move order. + // + // But truncates and sext/zext are always going to be fairly common from llvm. + // We have several options for how to deal with them: + // - Wherever possible combine them into an instruction that makes them + // "free". This includes loads/stores, which can perform the trunc as part + // of the memory operation. Or certain shuffles that can be turned into + // VMOVN/VMOVL. + // - Lane Interleaving to transform blocks surrounded by ext/trunc. So + // trunc(mul(sext(a), sext(b))) may become + // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in + // this case can use VMULL). This is performed in the + // MVELaneInterleavingPass. + // - Otherwise we have an option. By default we would expand the + // zext/sext/trunc into a series of lane extract/inserts going via GPR + // registers. One for each vector lane in the vector. This can obviously be + // very expensive. + // - The other option is to use the fact that loads/store can extend/truncate + // to turn a trunc into two truncating stack stores and a stack reload. This + // becomes 3 back-to-back memory operations, but at least that is less than + // all the insert/extracts. + // + // In order to do the last, we convert certain trunc's into MVETRUNC, which + // are either optimized where they can be, or eventually lowered into stack + // stores/loads. This prevents us from splitting a v8i16 trunc into two stores + // two early, where other instructions would be better, and stops us from + // having to reconstruct multiple buildvector shuffles into loads/stores. + if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8) + return SDValue(); + EVT FromVT = N->getOperand(0).getValueType(); + if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16) + return SDValue(); + + SDValue Lo, Hi; + std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); + SDLoc DL(N); + return DAG.getNode(ARMISD::MVETRUNC, DL, ToVT, Lo, Hi); +} + +static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, + const ARMSubtarget *Subtarget) { + if (!Subtarget->hasMVEIntegerOps()) + return SDValue(); + + // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC. + + EVT ToVT = N->getValueType(0); + if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16) + return SDValue(); + SDValue Op = N->getOperand(0); + EVT FromVT = Op.getValueType(); + if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8) + return SDValue(); + + SDLoc DL(N); + EVT ExtVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext()); + if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) + ExtVT = MVT::v8i16; + + unsigned Opcode = + N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT; + SDValue Ext = DAG.getNode(Opcode, DL, DAG.getVTList(ExtVT, ExtVT), Op); + SDValue Ext1 = Ext.getValue(1); + + if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) { + Ext = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext); + Ext1 = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext1); + } + + return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Ext, Ext1); +} + /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each /// element has been zero/sign-extended, depending on the isSigned parameter, /// from an integer type half its size. @@ -9394,13 +9779,20 @@ static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) { // the bottom bits of the predicate. // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect // for BE). + // Speaking of BE, apparently the rest of llvm will assume a reverse order to + // a natural VMSR(load), so needs to be reversed. SDLoc dl(Op); SDValue Load = DAG.getExtLoad( ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(), EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()), LD->getMemOperand()); - SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Load); + SDValue Val = Load; + if (DAG.getDataLayout().isBigEndian()) + Val = DAG.getNode(ISD::SRL, dl, MVT::i32, + DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load), + DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32)); + SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Val); if (MemVT != MVT::v16i1) Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred, DAG.getConstant(0, dl, MVT::i32)); @@ -9441,14 +9833,22 @@ static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) { SDValue Build = ST->getValue(); if (MemVT != MVT::v16i1) { SmallVector<SDValue, 16> Ops; - for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) + for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) { + unsigned Elt = DAG.getDataLayout().isBigEndian() + ? MemVT.getVectorNumElements() - I - 1 + : I; Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build, - DAG.getConstant(I, dl, MVT::i32))); + DAG.getConstant(Elt, dl, MVT::i32))); + } for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++) Ops.push_back(DAG.getUNDEF(MVT::i32)); Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops); } SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build); + if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian()) + GRP = DAG.getNode(ISD::SRL, dl, MVT::i32, + DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP), + DAG.getConstant(16, dl, MVT::i32)); return DAG.getTruncStore( ST->getChain(), dl, GRP, ST->getBasePtr(), EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()), @@ -9594,7 +9994,7 @@ static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, } static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) { - if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering())) + if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getSuccessOrdering())) // Acquire/Release load/store is not legal for targets without a dmb or // equivalent available. return SDValue(); @@ -9770,8 +10170,11 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget); case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget); - case ISD::TRUNCATE: return LowerTruncatei1(Op, DAG, Subtarget); + case ISD::TRUNCATE: return LowerTruncate(Op.getNode(), DAG, Subtarget); + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget); case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); + case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG); case ISD::MUL: return LowerMUL(Op, DAG); case ISD::SDIV: if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) @@ -9791,7 +10194,9 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerUnsignedALUO(Op, DAG); case ISD::SADDSAT: case ISD::SSUBSAT: - return LowerSADDSUBSAT(Op, DAG, Subtarget); + case ISD::UADDSAT: + case ISD::USUBSAT: + return LowerADDSUBSAT(Op, DAG, Subtarget); case ISD::LOAD: return LowerPredicateLoad(Op, DAG); case ISD::STORE: @@ -9891,7 +10296,9 @@ void ARMTargetLowering::ReplaceNodeResults(SDNode *N, return; case ISD::SADDSAT: case ISD::SSUBSAT: - Res = LowerSADDSUBSAT(SDValue(N, 0), DAG, Subtarget); + case ISD::UADDSAT: + case ISD::USUBSAT: + Res = LowerADDSUBSAT(SDValue(N, 0), DAG, Subtarget); break; case ISD::READCYCLECOUNTER: ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget); @@ -9912,6 +10319,13 @@ void ARMTargetLowering::ReplaceNodeResults(SDNode *N, case ISD::LOAD: LowerLOAD(N, Results, DAG); break; + case ISD::TRUNCATE: + Res = LowerTruncate(N, DAG, Subtarget); + break; + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + Res = LowerVectorExtend(N, DAG, Subtarget); + break; } if (Res.getNode()) Results.push_back(Res); @@ -10979,6 +11393,145 @@ static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, return true; } +/// Adds logic in loop entry MBB to calculate loop iteration count and adds +/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop +static Register genTPEntry(MachineBasicBlock *TpEntry, + MachineBasicBlock *TpLoopBody, + MachineBasicBlock *TpExit, Register OpSizeReg, + const TargetInstrInfo *TII, DebugLoc Dl, + MachineRegisterInfo &MRI) { + // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4. + Register AddDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + BuildMI(TpEntry, Dl, TII->get(ARM::t2ADDri), AddDestReg) + .addUse(OpSizeReg) + .addImm(15) + .add(predOps(ARMCC::AL)) + .addReg(0); + + Register LsrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + BuildMI(TpEntry, Dl, TII->get(ARM::t2LSRri), LsrDestReg) + .addUse(AddDestReg, RegState::Kill) + .addImm(4) + .add(predOps(ARMCC::AL)) + .addReg(0); + + Register TotalIterationsReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass); + BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg) + .addUse(LsrDestReg, RegState::Kill); + + BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopStart)) + .addUse(TotalIterationsReg) + .addMBB(TpExit); + + BuildMI(TpEntry, Dl, TII->get(ARM::t2B)) + .addMBB(TpLoopBody) + .add(predOps(ARMCC::AL)); + + return TotalIterationsReg; +} + +/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and +/// t2DoLoopEnd. These are used by later passes to generate tail predicated +/// loops. +static void genTPLoopBody(MachineBasicBlock *TpLoopBody, + MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, + const TargetInstrInfo *TII, DebugLoc Dl, + MachineRegisterInfo &MRI, Register OpSrcReg, + Register OpDestReg, Register ElementCountReg, + Register TotalIterationsReg, bool IsMemcpy) { + // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest + // array, loop iteration counter, predication counter. + + Register SrcPhiReg, CurrSrcReg; + if (IsMemcpy) { + // Current position in the src array + SrcPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + CurrSrcReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), SrcPhiReg) + .addUse(OpSrcReg) + .addMBB(TpEntry) + .addUse(CurrSrcReg) + .addMBB(TpLoopBody); + } + + // Current position in the dest array + Register DestPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + Register CurrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), DestPhiReg) + .addUse(OpDestReg) + .addMBB(TpEntry) + .addUse(CurrDestReg) + .addMBB(TpLoopBody); + + // Current loop counter + Register LoopCounterPhiReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass); + Register RemainingLoopIterationsReg = + MRI.createVirtualRegister(&ARM::GPRlrRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), LoopCounterPhiReg) + .addUse(TotalIterationsReg) + .addMBB(TpEntry) + .addUse(RemainingLoopIterationsReg) + .addMBB(TpLoopBody); + + // Predication counter + Register PredCounterPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + Register RemainingElementsReg = MRI.createVirtualRegister(&ARM::rGPRRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), PredCounterPhiReg) + .addUse(ElementCountReg) + .addMBB(TpEntry) + .addUse(RemainingElementsReg) + .addMBB(TpLoopBody); + + // Pass predication counter to VCTP + Register VccrReg = MRI.createVirtualRegister(&ARM::VCCRRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VCTP8), VccrReg) + .addUse(PredCounterPhiReg) + .addImm(ARMVCC::None) + .addReg(0); + + BuildMI(TpLoopBody, Dl, TII->get(ARM::t2SUBri), RemainingElementsReg) + .addUse(PredCounterPhiReg) + .addImm(16) + .add(predOps(ARMCC::AL)) + .addReg(0); + + // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR + Register SrcValueReg; + if (IsMemcpy) { + SrcValueReg = MRI.createVirtualRegister(&ARM::MQPRRegClass); + BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VLDRBU8_post)) + .addDef(CurrSrcReg) + .addDef(SrcValueReg) + .addReg(SrcPhiReg) + .addImm(16) + .addImm(ARMVCC::Then) + .addUse(VccrReg); + } else + SrcValueReg = OpSrcReg; + + BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VSTRBU8_post)) + .addDef(CurrDestReg) + .addUse(SrcValueReg) + .addReg(DestPhiReg) + .addImm(16) + .addImm(ARMVCC::Then) + .addUse(VccrReg); + + // Add the pseudoInstrs for decrementing the loop counter and marking the + // end:t2DoLoopDec and t2DoLoopEnd + BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg) + .addUse(LoopCounterPhiReg) + .addImm(1); + + BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopEnd)) + .addUse(RemainingLoopIterationsReg) + .addMBB(TpLoopBody); + + BuildMI(TpLoopBody, Dl, TII->get(ARM::t2B)) + .addMBB(TpExit) + .add(predOps(ARMCC::AL)); +} + MachineBasicBlock * ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const { @@ -11005,6 +11558,98 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } + case ARM::MVE_MEMCPYLOOPINST: + case ARM::MVE_MEMSETLOOPINST: { + + // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo + // into a Tail Predicated (TP) Loop. It adds the instructions to calculate + // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and + // adds the relevant instructions in the TP loop Body for generation of a + // WLSTP loop. + + // Below is relevant portion of the CFG after the transformation. + // The Machine Basic Blocks are shown along with branch conditions (in + // brackets). Note that TP entry/exit MBBs depict the entry/exit of this + // portion of the CFG and may not necessarily be the entry/exit of the + // function. + + // (Relevant) CFG after transformation: + // TP entry MBB + // | + // |-----------------| + // (n <= 0) (n > 0) + // | | + // | TP loop Body MBB<--| + // | | | + // \ |___________| + // \ / + // TP exit MBB + + MachineFunction *MF = BB->getParent(); + MachineFunctionProperties &Properties = MF->getProperties(); + MachineRegisterInfo &MRI = MF->getRegInfo(); + + Register OpDestReg = MI.getOperand(0).getReg(); + Register OpSrcReg = MI.getOperand(1).getReg(); + Register OpSizeReg = MI.getOperand(2).getReg(); + + // Allocate the required MBBs and add to parent function. + MachineBasicBlock *TpEntry = BB; + MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock(); + MachineBasicBlock *TpExit; + + MF->push_back(TpLoopBody); + + // If any instructions are present in the current block after + // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and + // move the instructions into the newly created exit block. If there are no + // instructions add an explicit branch to the FallThrough block and then + // split. + // + // The split is required for two reasons: + // 1) A terminator(t2WhileLoopStart) will be placed at that site. + // 2) Since a TPLoopBody will be added later, any phis in successive blocks + // need to be updated. splitAt() already handles this. + TpExit = BB->splitAt(MI, false); + if (TpExit == BB) { + assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the " + "block containing memcpy/memset Pseudo"); + TpExit = BB->getFallThrough(); + BuildMI(BB, dl, TII->get(ARM::t2B)) + .addMBB(TpExit) + .add(predOps(ARMCC::AL)); + TpExit = BB->splitAt(MI, false); + } + + // Add logic for iteration count + Register TotalIterationsReg = + genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, dl, MRI); + + // Add the vectorized (and predicated) loads/store instructions + bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST; + genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, dl, MRI, OpSrcReg, + OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy); + + // Required to avoid conflict with the MachineVerifier during testing. + Properties.reset(MachineFunctionProperties::Property::NoPHIs); + + // Connect the blocks + TpEntry->addSuccessor(TpLoopBody); + TpLoopBody->addSuccessor(TpLoopBody); + TpLoopBody->addSuccessor(TpExit); + + // Reorder for a more natural layout + TpLoopBody->moveAfter(TpEntry); + TpExit->moveAfter(TpLoopBody); + + // Finally, remove the memcpy Psuedo Instruction + MI.eraseFromParent(); + + // Return the exit block as it may contain other instructions requiring a + // custom inserter + return TpExit; + } + // The Thumb2 pre-indexed stores have the same MI operands, they just // define them differently in the .td files from the isel patterns, so // they need pseudos. @@ -11266,14 +11911,6 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return EmitLowered__chkstk(MI, BB); case ARM::WIN__DBZCHK: return EmitLowered__dbzchk(MI, BB); - case ARM::t2DoLoopStart: - // We are just here to set a register allocation hint, prefering lr for the - // input register to make it more likely to be movable and removable, later - // in the pipeline. - Register R = MI.getOperand(1).getReg(); - MachineFunction *MF = MI.getParent()->getParent(); - MF->getRegInfo().setRegAllocationHint(R, ARMRI::RegLR, 0); - return BB; } } @@ -12285,7 +12922,7 @@ static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) { Ext1.getOpcode() != ISD::SIGN_EXTEND) return SDValue(); EVT VecVT = Ext0.getOperand(0).getValueType(); - if (VecVT != MVT::v4i32 && VecVT != MVT::v8i16 && VecVT != MVT::v16i8) + if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1) return SDValue(); if (Ext1.getOperand(0).getValueType() != VecVT || VecVT.getScalarType() != ScalarType || @@ -12293,9 +12930,42 @@ static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) { return SDValue(); SDLoc DL(Mul); - SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, VecVT, Ext0.getOperand(0), - Ext1.getOperand(0)); - return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, VQDMULH); + unsigned LegalLanes = 128 / (ShftAmt + 1); + EVT LegalVecVT = MVT::getVectorVT(ScalarType, LegalLanes); + // For types smaller than legal vectors extend to be legal and only use needed + // lanes. + if (VecVT.getSizeInBits() < 128) { + EVT ExtVecVT = + MVT::getVectorVT(MVT::getIntegerVT(128 / VecVT.getVectorNumElements()), + VecVT.getVectorNumElements()); + SDValue Inp0 = + DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext0.getOperand(0)); + SDValue Inp1 = + DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext1.getOperand(0)); + Inp0 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp0); + Inp1 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp1); + SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1); + SDValue Trunc = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, ExtVecVT, VQDMULH); + Trunc = DAG.getNode(ISD::TRUNCATE, DL, VecVT, Trunc); + return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc); + } + + // For larger types, split into legal sized chunks. + assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type"); + unsigned NumParts = VecVT.getSizeInBits() / 128; + SmallVector<SDValue> Parts; + for (unsigned I = 0; I < NumParts; ++I) { + SDValue Inp0 = + DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext0.getOperand(0), + DAG.getVectorIdxConstant(I * LegalLanes, DL)); + SDValue Inp1 = + DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext1.getOperand(0), + DAG.getVectorIdxConstant(I * LegalLanes, DL)); + SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1); + Parts.push_back(VQDMULH); + } + return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, + DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Parts)); } static SDValue PerformVSELECTCombine(SDNode *N, @@ -12394,8 +13064,7 @@ static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, return SDValue(); } -static SDValue PerformADDVecReduce(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI, +static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { if (!Subtarget->hasMVEIntegerOps() || N->getValueType(0) != MVT::i64) return SDValue(); @@ -12408,28 +13077,39 @@ static SDValue PerformADDVecReduce(SDNode *N, // t1: i32,i32 = ARMISD::VADDLVs x // t2: i64 = build_pair t1, t1:1 // t3: i64 = add t2, y + // Otherwise we try to push the add up above VADDLVAx, to potentially allow + // the add to be simplified seperately. // We also need to check for sext / zext and commutitive adds. auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA, SDValue NB) { if (NB->getOpcode() != ISD::BUILD_PAIR) return SDValue(); SDValue VecRed = NB->getOperand(0); - if (VecRed->getOpcode() != Opcode || VecRed.getResNo() != 0 || + if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) || + VecRed.getResNo() != 0 || NB->getOperand(1) != SDValue(VecRed.getNode(), 1)) return SDValue(); SDLoc dl(N); + if (VecRed->getOpcode() == OpcodeA) { + // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y) + SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, + VecRed.getOperand(0), VecRed.getOperand(1)); + NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA); + } + SmallVector<SDValue, 4> Ops; - Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA, - DCI.DAG.getConstant(0, dl, MVT::i32))); - Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA, - DCI.DAG.getConstant(1, dl, MVT::i32))); - for (unsigned i = 0, e = VecRed.getNumOperands(); i < e; i++) - Ops.push_back(VecRed->getOperand(i)); - SDValue Red = DCI.DAG.getNode(OpcodeA, dl, - DCI.DAG.getVTList({MVT::i32, MVT::i32}), Ops); - return DCI.DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red, - SDValue(Red.getNode(), 1)); + Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA, + DAG.getConstant(0, dl, MVT::i32))); + Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA, + DAG.getConstant(1, dl, MVT::i32))); + unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0; + for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++) + Ops.push_back(VecRed->getOperand(I)); + SDValue Red = + DAG.getNode(OpcodeA, dl, DAG.getVTList({MVT::i32, MVT::i32}), Ops); + return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red, + SDValue(Red.getNode(), 1)); }; if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1)) @@ -12638,7 +13318,7 @@ static SDValue PerformADDCombine(SDNode *N, if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget)) return Result; - if (SDValue Result = PerformADDVecReduce(N, DCI, Subtarget)) + if (SDValue Result = PerformADDVecReduce(N, DCI.DAG, Subtarget)) return Result; // First try with the default operand order. @@ -12649,6 +13329,26 @@ static SDValue PerformADDCombine(SDNode *N, return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget); } +// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC) +// providing -X is as cheap as X (currently, just a constant). +static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) { + if (N->getValueType(0) != MVT::i32 || !isNullConstant(N->getOperand(0))) + return SDValue(); + SDValue CSINC = N->getOperand(1); + if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse()) + return SDValue(); + + ConstantSDNode *X = dyn_cast<ConstantSDNode>(CSINC.getOperand(0)); + if (!X) + return SDValue(); + + return DAG.getNode(ARMISD::CSINV, SDLoc(N), MVT::i32, + DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0), + CSINC.getOperand(0)), + CSINC.getOperand(1), CSINC.getOperand(2), + CSINC.getOperand(3)); +} + /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB. /// static SDValue PerformSUBCombine(SDNode *N, @@ -12662,6 +13362,9 @@ static SDValue PerformSUBCombine(SDNode *N, if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI)) return Result; + if (SDValue R = PerformSubCSINCCombine(N, DCI.DAG)) + return R; + if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(0).isVector()) return SDValue(); @@ -13254,8 +13957,7 @@ static bool CanInvertMVEVCMP(SDValue N) { return isValidMVECond(CC, N->getOperand(0).getValueType().isFloatingPoint()); } -static SDValue PerformORCombine_i1(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI, +static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain // together with predicates @@ -13274,10 +13976,10 @@ static SDValue PerformORCombine_i1(SDNode *N, if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1))) return SDValue(); - SDValue NewN0 = DCI.DAG.getLogicalNOT(DL, N0, VT); - SDValue NewN1 = DCI.DAG.getLogicalNOT(DL, N1, VT); - SDValue And = DCI.DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1); - return DCI.DAG.getLogicalNOT(DL, And, VT); + SDValue NewN0 = DAG.getLogicalNOT(DL, N0, VT); + SDValue NewN1 = DAG.getLogicalNOT(DL, N1, VT); + SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1); + return DAG.getLogicalNOT(DL, And, VT); } /// PerformORCombine - Target-specific dag combine xforms for ISD::OR @@ -13295,7 +13997,7 @@ static SDValue PerformORCombine(SDNode *N, if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)) - return PerformORCombine_i1(N, DCI, Subtarget); + return PerformORCombine_i1(N, DAG, Subtarget); APInt SplatBits, SplatUndef; unsigned SplatBitSize; @@ -13414,8 +14116,8 @@ static SDValue PerformXORCombine(SDNode *N, Ops.push_back(N0->getOperand(0)); if (N0->getOpcode() == ARMISD::VCMP) Ops.push_back(N0->getOperand(1)); - Ops.push_back(DCI.DAG.getConstant(CC, DL, MVT::i32)); - return DCI.DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops); + Ops.push_back(DAG.getConstant(CC, DL, MVT::i32)); + return DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops); } } } @@ -13456,52 +14158,40 @@ static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) { } static SDValue FindBFIToCombineWith(SDNode *N) { - // We have a BFI in N. Follow a possible chain of BFIs and find a BFI it can combine with, - // if one exists. + // We have a BFI in N. Find a BFI it can combine with, if one exists. APInt ToMask, FromMask; SDValue From = ParseBFI(N, ToMask, FromMask); SDValue To = N->getOperand(0); - // Now check for a compatible BFI to merge with. We can pass through BFIs that - // aren't compatible, but not if they set the same bit in their destination as - // we do (or that of any BFI we're going to combine with). SDValue V = To; - APInt CombinedToMask = ToMask; - while (V.getOpcode() == ARMISD::BFI) { - APInt NewToMask, NewFromMask; - SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask); - if (NewFrom != From) { - // This BFI has a different base. Keep going. - CombinedToMask |= NewToMask; - V = V.getOperand(0); - continue; - } + if (V.getOpcode() != ARMISD::BFI) + return SDValue(); - // Do the written bits conflict with any we've seen so far? - if ((NewToMask & CombinedToMask).getBoolValue()) - // Conflicting bits - bail out because going further is unsafe. - return SDValue(); + APInt NewToMask, NewFromMask; + SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask); + if (NewFrom != From) + return SDValue(); - // Are the new bits contiguous when combined with the old bits? - if (BitsProperlyConcatenate(ToMask, NewToMask) && - BitsProperlyConcatenate(FromMask, NewFromMask)) - return V; - if (BitsProperlyConcatenate(NewToMask, ToMask) && - BitsProperlyConcatenate(NewFromMask, FromMask)) - return V; + // Do the written bits conflict with any we've seen so far? + if ((NewToMask & ToMask).getBoolValue()) + // Conflicting bits. + return SDValue(); - // We've seen a write to some bits, so track it. - CombinedToMask |= NewToMask; - // Keep going... - V = V.getOperand(0); - } + // Are the new bits contiguous when combined with the old bits? + if (BitsProperlyConcatenate(ToMask, NewToMask) && + BitsProperlyConcatenate(FromMask, NewFromMask)) + return V; + if (BitsProperlyConcatenate(NewToMask, ToMask) && + BitsProperlyConcatenate(NewFromMask, FromMask)) + return V; return SDValue(); } -static SDValue PerformBFICombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI) { +static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) { + SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); + if (N1.getOpcode() == ISD::AND) { // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff // the bits being cleared by the AND are not demanded by the BFI. @@ -13517,17 +14207,13 @@ static SDValue PerformBFICombine(SDNode *N, unsigned Mask = (1u << Width) - 1; unsigned Mask2 = N11C->getZExtValue(); if ((Mask & (~Mask2)) == 0) - return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0), - N->getOperand(0), N1.getOperand(0), - N->getOperand(2)); - } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) { - // We have a BFI of a BFI. Walk up the BFI chain to see how long it goes. - // Keep track of any consecutive bits set that all come from the same base - // value. We can combine these together into a single BFI. - SDValue CombineBFI = FindBFIToCombineWith(N); - if (CombineBFI == SDValue()) - return SDValue(); + return DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0), + N->getOperand(0), N1.getOperand(0), N->getOperand(2)); + return SDValue(); + } + // Look for another BFI to combine with. + if (SDValue CombineBFI = FindBFIToCombineWith(N)) { // We've found a BFI. APInt ToMask1, FromMask1; SDValue From1 = ParseBFI(N, ToMask1, FromMask1); @@ -13537,9 +14223,7 @@ static SDValue PerformBFICombine(SDNode *N, assert(From1 == From2); (void)From2; - // First, unlink CombineBFI. - DCI.DAG.ReplaceAllUsesWith(CombineBFI, CombineBFI.getOperand(0)); - // Then create a new BFI, combining the two together. + // Create a new BFI, combining the two together. APInt NewFromMask = FromMask1 | FromMask2; APInt NewToMask = ToMask1 | ToMask2; @@ -13547,12 +14231,33 @@ static SDValue PerformBFICombine(SDNode *N, SDLoc dl(N); if (NewFromMask[0] == 0) - From1 = DCI.DAG.getNode( - ISD::SRL, dl, VT, From1, - DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT)); - return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1, - DCI.DAG.getConstant(~NewToMask, dl, VT)); + From1 = DAG.getNode( + ISD::SRL, dl, VT, From1, + DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT)); + return DAG.getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1, + DAG.getConstant(~NewToMask, dl, VT)); } + + // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so + // that lower bit insertions are performed first, providing that M1 and M2 + // do no overlap. This can allow multiple BFI instructions to be combined + // together by the other folds above. + if (N->getOperand(0).getOpcode() == ARMISD::BFI) { + APInt ToMask1 = ~N->getConstantOperandAPInt(2); + APInt ToMask2 = ~N0.getConstantOperandAPInt(2); + + if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 || + ToMask1.countLeadingZeros() < ToMask2.countLeadingZeros()) + return SDValue(); + + EVT VT = N->getValueType(0); + SDLoc dl(N); + SDValue BFI1 = DAG.getNode(ARMISD::BFI, dl, VT, N0.getOperand(0), + N->getOperand(1), N->getOperand(2)); + return DAG.getNode(ARMISD::BFI, dl, VT, BFI1, N0.getOperand(1), + N0.getOperand(2)); + } + return SDValue(); } @@ -13597,6 +14302,54 @@ static SDValue PerformVMOVRRDCombine(SDNode *N, return Result; } + // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d + // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b + if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT && + isa<ConstantSDNode>(InDouble.getOperand(1))) { + SDValue BV = InDouble.getOperand(0); + // Look up through any nop bitcasts and vector_reg_casts. bitcasts may + // change lane order under big endian. + bool BVSwap = BV.getOpcode() == ISD::BITCAST; + while ( + (BV.getOpcode() == ISD::BITCAST || + BV.getOpcode() == ARMISD::VECTOR_REG_CAST) && + (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) { + BVSwap = BV.getOpcode() == ISD::BITCAST; + BV = BV.getOperand(0); + } + if (BV.getValueType() != MVT::v4i32) + return SDValue(); + + // Handle buildvectors, pulling out the correct lane depending on + // endianness. + unsigned Offset = InDouble.getConstantOperandVal(1) == 1 ? 2 : 0; + if (BV.getOpcode() == ISD::BUILD_VECTOR) { + SDValue Op0 = BV.getOperand(Offset); + SDValue Op1 = BV.getOperand(Offset + 1); + if (!Subtarget->isLittle() && BVSwap) + std::swap(Op0, Op1); + + return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N)); + } + + // A chain of insert_vectors, grabbing the correct value of the chain of + // inserts. + SDValue Op0, Op1; + while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) { + if (isa<ConstantSDNode>(BV.getOperand(2))) { + if (BV.getConstantOperandVal(2) == Offset) + Op0 = BV.getOperand(1); + if (BV.getConstantOperandVal(2) == Offset + 1) + Op1 = BV.getOperand(1); + } + BV = BV.getOperand(0); + } + if (!Subtarget->isLittle() && BVSwap) + std::swap(Op0, Op1); + if (Op0 && Op1) + return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N)); + } + return SDValue(); } @@ -13618,7 +14371,8 @@ static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) { return SDValue(); } -static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { +static SDValue PerformVMOVhrCombine(SDNode *N, + TargetLowering::DAGCombinerInfo &DCI) { SDValue Op0 = N->getOperand(0); // VMOVhr (VMOVrh (X)) -> X @@ -13664,15 +14418,14 @@ static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo & return SDValue(); } -static SDValue PerformVMOVrhCombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI) { +static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); // fold (VMOVrh (fpconst x)) -> const x if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N0)) { APFloat V = C->getValueAPF(); - return DCI.DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT); + return DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT); } // fold (VMOVrh (load x)) -> (zextload (i16*)x) @@ -13680,18 +14433,18 @@ static SDValue PerformVMOVrhCombine(SDNode *N, LoadSDNode *LN0 = cast<LoadSDNode>(N0); SDValue Load = - DCI.DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(), - LN0->getBasePtr(), MVT::i16, LN0->getMemOperand()); - DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0)); - DCI.DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); + DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(), + LN0->getBasePtr(), MVT::i16, LN0->getMemOperand()); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0)); + DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); return Load; } // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n) if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT && isa<ConstantSDNode>(N0->getOperand(1))) - return DCI.DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0), - N0->getOperand(1)); + return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0), + N0->getOperand(1)); return SDValue(); } @@ -13869,30 +14622,32 @@ PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { return SDValue(); } -static SDValue -PerformVECTOR_REG_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, - const ARMSubtarget *ST) { +static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, + const ARMSubtarget *ST) { EVT VT = N->getValueType(0); SDValue Op = N->getOperand(0); SDLoc dl(N); // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST if (ST->isLittle()) - return DCI.DAG.getNode(ISD::BITCAST, dl, VT, Op); + return DAG.getNode(ISD::BITCAST, dl, VT, Op); + + // VECTOR_REG_CAST undef -> undef + if (Op.isUndef()) + return DAG.getUNDEF(VT); // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x) if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) { // If the valuetypes are the same, we can remove the cast entirely. if (Op->getOperand(0).getValueType() == VT) return Op->getOperand(0); - return DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0)); + return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0)); } return SDValue(); } -static SDValue PerformVCMPCombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI, +static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { if (!Subtarget->hasMVEIntegerOps()) return SDValue(); @@ -13906,19 +14661,18 @@ static SDValue PerformVCMPCombine(SDNode *N, // vcmp X, 0, cc -> vcmpz X, cc if (isZeroVector(Op1)) - return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, - N->getOperand(2)); + return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2)); unsigned SwappedCond = getSwappedCondition(Cond); if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) { // vcmp 0, X, cc -> vcmpz X, reversed(cc) if (isZeroVector(Op0)) - return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1, - DCI.DAG.getConstant(SwappedCond, dl, MVT::i32)); + return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1, + DAG.getConstant(SwappedCond, dl, MVT::i32)); // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc) if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP) - return DCI.DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0, - DCI.DAG.getConstant(SwappedCond, dl, MVT::i32)); + return DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0, + DAG.getConstant(SwappedCond, dl, MVT::i32)); } return SDValue(); @@ -13950,8 +14704,73 @@ static SDValue PerformInsertEltCombine(SDNode *N, return DAG.getNode(ISD::BITCAST, dl, VT, InsElt); } +// Convert a pair of extracts from the same base vector to a VMOVRRD. Either +// directly or bitcast to an integer if the original is a float vector. +// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2) +// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2) +static SDValue +PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { + EVT VT = N->getValueType(0); + SDLoc dl(N); + + if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 || + !DCI.DAG.getTargetLoweringInfo().isTypeLegal(MVT::f64)) + return SDValue(); + + SDValue Ext = SDValue(N, 0); + if (Ext.getOpcode() == ISD::BITCAST && + Ext.getOperand(0).getValueType() == MVT::f32) + Ext = Ext.getOperand(0); + if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + !isa<ConstantSDNode>(Ext.getOperand(1)) || + Ext.getConstantOperandVal(1) % 2 != 0) + return SDValue(); + if (Ext->use_size() == 1 && + (Ext->use_begin()->getOpcode() == ISD::SINT_TO_FP || + Ext->use_begin()->getOpcode() == ISD::UINT_TO_FP)) + return SDValue(); + + SDValue Op0 = Ext.getOperand(0); + EVT VecVT = Op0.getValueType(); + unsigned Lane = Ext.getConstantOperandVal(1); + if (VecVT.getVectorNumElements() != 4) + return SDValue(); + + // Find another extract, of Lane + 1 + auto OtherIt = find_if(Op0->uses(), [&](SDNode *V) { + return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT && + isa<ConstantSDNode>(V->getOperand(1)) && + V->getConstantOperandVal(1) == Lane + 1; + }); + if (OtherIt == Op0->uses().end()) + return SDValue(); + + // For float extracts, we need to be converting to a i32 for both vector + // lanes. + SDValue OtherExt(*OtherIt, 0); + if (OtherExt.getValueType() != MVT::i32) { + if (OtherExt->use_size() != 1 || + OtherExt->use_begin()->getOpcode() != ISD::BITCAST || + OtherExt->use_begin()->getValueType(0) != MVT::i32) + return SDValue(); + OtherExt = SDValue(*OtherExt->use_begin(), 0); + } + + // Convert the type to a f64 and extract with a VMOVRRD. + SDValue F64 = DCI.DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, + DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0), + DCI.DAG.getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32)); + SDValue VMOVRRD = + DCI.DAG.getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32}, F64); + + DCI.CombineTo(OtherExt.getNode(), SDValue(VMOVRRD.getNode(), 1)); + return VMOVRRD; +} + static SDValue PerformExtractEltCombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI) { + TargetLowering::DAGCombinerInfo &DCI, + const ARMSubtarget *ST) { SDValue Op0 = N->getOperand(0); EVT VT = N->getValueType(0); SDLoc dl(N); @@ -13963,6 +14782,8 @@ static SDValue PerformExtractEltCombine(SDNode *N, return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X); if (VT == MVT::i32 && X.getValueType() == MVT::f16) return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X); + if (VT == MVT::f32 && X.getValueType() == MVT::i32) + return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X); while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST) X = X->getOperand(0); @@ -13970,12 +14791,131 @@ static SDValue PerformExtractEltCombine(SDNode *N, return X; } + // extract ARM_BUILD_VECTOR -> x + if (Op0->getOpcode() == ARMISD::BUILD_VECTOR && + isa<ConstantSDNode>(N->getOperand(1)) && + N->getConstantOperandVal(1) < Op0.getNumOperands()) { + return Op0.getOperand(N->getConstantOperandVal(1)); + } + + // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b + if (Op0.getValueType() == MVT::v4i32 && + isa<ConstantSDNode>(N->getOperand(1)) && + Op0.getOpcode() == ISD::BITCAST && + Op0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && + Op0.getOperand(0).getValueType() == MVT::v2f64) { + SDValue BV = Op0.getOperand(0); + unsigned Offset = N->getConstantOperandVal(1); + SDValue MOV = BV.getOperand(Offset < 2 ? 0 : 1); + if (MOV.getOpcode() == ARMISD::VMOVDRR) + return MOV.getOperand(ST->isLittle() ? Offset % 2 : 1 - Offset % 2); + } + + // extract x, n; extract x, n+1 -> VMOVRRD x + if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI)) + return R; + + // extract (MVETrunc(x)) -> extract x + if (Op0->getOpcode() == ARMISD::MVETRUNC) { + unsigned Idx = N->getConstantOperandVal(1); + unsigned Vec = + Idx / Op0->getOperand(0).getValueType().getVectorNumElements(); + unsigned SubIdx = + Idx % Op0->getOperand(0).getValueType().getVectorNumElements(); + return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec), + DCI.DAG.getConstant(SubIdx, dl, MVT::i32)); + } + + return SDValue(); +} + +static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) { + SDValue Op = N->getOperand(0); + EVT VT = N->getValueType(0); + + // sext_inreg(VGETLANEu) -> VGETLANEs + if (Op.getOpcode() == ARMISD::VGETLANEu && + cast<VTSDNode>(N->getOperand(1))->getVT() == + Op.getOperand(0).getValueType().getScalarType()) + return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0), + Op.getOperand(1)); + + return SDValue(); +} + +// When lowering complex nodes that we recognize, like VQDMULH and MULH, we +// can end up with shuffle(binop(shuffle, shuffle)), that can be simplified to +// binop as the shuffles cancel out. +static SDValue FlattenVectorShuffle(ShuffleVectorSDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + if (!N->getOperand(1).isUndef() || N->getOperand(0).getValueType() != VT) + return SDValue(); + SDValue Op = N->getOperand(0); + + // Looking for binary operators that will have been folded from + // truncates/extends. + switch (Op.getOpcode()) { + case ARMISD::VQDMULH: + case ISD::MULHS: + case ISD::MULHU: + case ISD::ABDS: + case ISD::ABDU: + break; + default: + return SDValue(); + } + + ShuffleVectorSDNode *Op0 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(0)); + ShuffleVectorSDNode *Op1 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(1)); + if (!Op0 || !Op1 || !Op0->getOperand(1).isUndef() || + !Op1->getOperand(1).isUndef() || Op0->getMask() != Op1->getMask() || + Op0->getOperand(0).getValueType() != VT) + return SDValue(); + + // Check the mask turns into an identity shuffle. + ArrayRef<int> NMask = N->getMask(); + ArrayRef<int> OpMask = Op0->getMask(); + for (int i = 0, e = NMask.size(); i != e; i++) { + if (NMask[i] > 0 && OpMask[NMask[i]] > 0 && OpMask[NMask[i]] != i) + return SDValue(); + } + + return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), + Op0->getOperand(0), Op1->getOperand(0)); +} + +// shuffle(MVETrunc(x, y)) -> VMOVN(x, y) +static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, + SelectionDAG &DAG) { + SDValue Trunc = N->getOperand(0); + EVT VT = Trunc.getValueType(); + if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(1).isUndef()) + return SDValue(); + + SDLoc DL(Trunc); + if (isVMOVNTruncMask(N->getMask(), VT, 0)) + return DAG.getNode( + ARMISD::VMOVN, DL, VT, + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)), + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)), + DAG.getConstant(1, DL, MVT::i32)); + else if (isVMOVNTruncMask(N->getMask(), VT, 1)) + return DAG.getNode( + ARMISD::VMOVN, DL, VT, + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)), + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)), + DAG.getConstant(1, DL, MVT::i32)); return SDValue(); } /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for /// ISD::VECTOR_SHUFFLE. static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) { + if (SDValue R = FlattenVectorShuffle(cast<ShuffleVectorSDNode>(N), DAG)) + return R; + if (SDValue R = PerformShuffleVMOVNCombine(cast<ShuffleVectorSDNode>(N), DAG)) + return R; + // The LLVM shufflevector instruction does not require the shuffle mask // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the @@ -14064,6 +15004,9 @@ static SDValue CombineBaseUpdate(SDNode *N, // Find the new opcode for the updating load/store. bool isLoadOp = true; bool isLaneOp = false; + // Workaround for vst1x and vld1x intrinsics which do not have alignment + // as an operand. + bool hasAlignment = true; unsigned NewOpc = 0; unsigned NumVecs = 0; if (isIntrinsic) { @@ -14078,15 +15021,18 @@ static SDValue CombineBaseUpdate(SDNode *N, NumVecs = 3; break; case Intrinsic::arm_neon_vld4: NewOpc = ARMISD::VLD4_UPD; NumVecs = 4; break; - case Intrinsic::arm_neon_vld1x2: - case Intrinsic::arm_neon_vld1x3: - case Intrinsic::arm_neon_vld1x4: - case Intrinsic::arm_neon_vld2dup: - case Intrinsic::arm_neon_vld3dup: - case Intrinsic::arm_neon_vld4dup: - // TODO: Support updating VLD1x and VLDxDUP nodes. For now, we just skip - // combining base updates for such intrinsics. - continue; + case Intrinsic::arm_neon_vld1x2: NewOpc = ARMISD::VLD1x2_UPD; + NumVecs = 2; hasAlignment = false; break; + case Intrinsic::arm_neon_vld1x3: NewOpc = ARMISD::VLD1x3_UPD; + NumVecs = 3; hasAlignment = false; break; + case Intrinsic::arm_neon_vld1x4: NewOpc = ARMISD::VLD1x4_UPD; + NumVecs = 4; hasAlignment = false; break; + case Intrinsic::arm_neon_vld2dup: NewOpc = ARMISD::VLD2DUP_UPD; + NumVecs = 2; break; + case Intrinsic::arm_neon_vld3dup: NewOpc = ARMISD::VLD3DUP_UPD; + NumVecs = 3; break; + case Intrinsic::arm_neon_vld4dup: NewOpc = ARMISD::VLD4DUP_UPD; + NumVecs = 4; break; case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD; NumVecs = 2; isLaneOp = true; break; case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD; @@ -14107,6 +15053,12 @@ static SDValue CombineBaseUpdate(SDNode *N, NumVecs = 3; isLoadOp = false; isLaneOp = true; break; case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD; NumVecs = 4; isLoadOp = false; isLaneOp = true; break; + case Intrinsic::arm_neon_vst1x2: NewOpc = ARMISD::VST1x2_UPD; + NumVecs = 2; isLoadOp = false; hasAlignment = false; break; + case Intrinsic::arm_neon_vst1x3: NewOpc = ARMISD::VST1x3_UPD; + NumVecs = 3; isLoadOp = false; hasAlignment = false; break; + case Intrinsic::arm_neon_vst1x4: NewOpc = ARMISD::VST1x4_UPD; + NumVecs = 4; isLoadOp = false; hasAlignment = false; break; } } else { isLaneOp = true; @@ -14134,8 +15086,12 @@ static SDValue CombineBaseUpdate(SDNode *N, VecTy = N->getOperand(1).getValueType(); } + bool isVLDDUPOp = + NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD || + NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD; + unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8; - if (isLaneOp) + if (isLaneOp || isVLDDUPOp) NumBytes /= VecTy.getVectorNumElements(); // If the increment is a constant, it must match the memory ref size. @@ -14210,7 +15166,9 @@ static SDValue CombineBaseUpdate(SDNode *N, } else { // Loads (and of course intrinsics) match the intrinsics' signature, // so just add all but the alignment operand. - for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i) + unsigned LastOperand = + hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands(); + for (unsigned i = AddrOpIdx + 1; i < LastOperand; ++i) Ops.push_back(N->getOperand(i)); } @@ -14505,10 +15463,8 @@ static SDValue PerformVDUPLANECombine(SDNode *N, } /// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP. -static SDValue PerformVDUPCombine(SDNode *N, - TargetLowering::DAGCombinerInfo &DCI, +static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { - SelectionDAG &DAG = DCI.DAG; SDValue Op = N->getOperand(0); SDLoc dl(N); @@ -14516,11 +15472,11 @@ static SDValue PerformVDUPCombine(SDNode *N, // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will // need to come from a GPR. if (Op.getValueType() == MVT::f32) - return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), - DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op)); + return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), + DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op)); else if (Op.getValueType() == MVT::f16) - return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), - DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op)); + return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), + DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op)); } if (!Subtarget->hasNEON()) @@ -14532,12 +15488,12 @@ static SDValue PerformVDUPCombine(SDNode *N, LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode()); if (LD && Op.hasOneUse() && LD->isUnindexed() && LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) { - SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1), - DAG.getConstant(LD->getAlignment(), SDLoc(N), MVT::i32) }; + SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1), + DAG.getConstant(LD->getAlignment(), SDLoc(N), MVT::i32)}; SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other); - SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys, - Ops, LD->getMemoryVT(), - LD->getMemOperand()); + SDValue VLDDup = + DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys, Ops, + LD->getMemoryVT(), LD->getMemOperand()); DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1)); return VLDDup; } @@ -14642,7 +15598,7 @@ static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); } -// Try taking a single vector store from an truncate (which would otherwise turn +// Try taking a single vector store from an fpround (which would otherwise turn // into an expensive buildvector) and splitting it into a series of narrowing // stores. static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, @@ -14650,7 +15606,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) return SDValue(); SDValue Trunc = St->getValue(); - if (Trunc->getOpcode() != ISD::TRUNCATE && Trunc->getOpcode() != ISD::FP_ROUND) + if (Trunc->getOpcode() != ISD::FP_ROUND) return SDValue(); EVT FromVT = Trunc->getOperand(0).getValueType(); EVT ToVT = Trunc.getValueType(); @@ -14660,16 +15616,11 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, EVT ToEltVT = ToVT.getVectorElementType(); EVT FromEltVT = FromVT.getVectorElementType(); - unsigned NumElements = 0; - if (FromEltVT == MVT::i32 && (ToEltVT == MVT::i16 || ToEltVT == MVT::i8)) - NumElements = 4; - if (FromEltVT == MVT::i16 && ToEltVT == MVT::i8) - NumElements = 8; - if (FromEltVT == MVT::f32 && ToEltVT == MVT::f16) - NumElements = 4; - if (NumElements == 0 || - (FromEltVT != MVT::f32 && FromVT.getVectorNumElements() == NumElements) || - FromVT.getVectorNumElements() % NumElements != 0) + if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16) + return SDValue(); + + unsigned NumElements = 4; + if (FromVT.getVectorNumElements() % NumElements != 0) return SDValue(); // Test if the Trunc will be convertable to a VMOVN with a shuffle, and if so @@ -14698,14 +15649,6 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, return true; }; - // It may be preferable to keep the store unsplit as the trunc may end up - // being removed. Check that here. - if (Trunc.getOperand(0).getOpcode() == ISD::SMIN) { - if (SDValue U = PerformVQDMULHCombine(Trunc.getOperand(0).getNode(), DAG)) { - DAG.ReplaceAllUsesWith(Trunc.getOperand(0), U); - return SDValue(); - } - } if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.getOperand(0))) if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true)) return SDValue(); @@ -14735,12 +15678,10 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0), DAG.getConstant(i * NumElements, DL, MVT::i32)); - if (ToEltVT == MVT::f16) { - SDValue FPTrunc = - DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16), - Extract, DAG.getConstant(0, DL, MVT::i32)); - Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc); - } + SDValue FPTrunc = + DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16), + Extract, DAG.getConstant(0, DL, MVT::i32)); + Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc); SDValue Store = DAG.getTruncStore( Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset), @@ -14750,6 +15691,83 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); } +// Try taking a single vector store from an MVETRUNC (which would otherwise turn +// into an expensive buildvector) and splitting it into a series of narrowing +// stores. +static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, + SelectionDAG &DAG) { + if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) + return SDValue(); + SDValue Trunc = St->getValue(); + if (Trunc->getOpcode() != ARMISD::MVETRUNC) + return SDValue(); + EVT FromVT = Trunc->getOperand(0).getValueType(); + EVT ToVT = Trunc.getValueType(); + + LLVMContext &C = *DAG.getContext(); + SDLoc DL(St); + // Details about the old store + SDValue Ch = St->getChain(); + SDValue BasePtr = St->getBasePtr(); + Align Alignment = St->getOriginalAlign(); + MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags(); + AAMDNodes AAInfo = St->getAAInfo(); + + EVT NewToVT = EVT::getVectorVT(C, ToVT.getVectorElementType(), + FromVT.getVectorNumElements()); + + SmallVector<SDValue, 4> Stores; + for (unsigned i = 0; i < Trunc.getNumOperands(); i++) { + unsigned NewOffset = + i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8; + SDValue NewPtr = + DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset)); + + SDValue Extract = Trunc.getOperand(i); + SDValue Store = DAG.getTruncStore( + Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset), + NewToVT, Alignment.value(), MMOFlags, AAInfo); + Stores.push_back(Store); + } + return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); +} + +// Given a floating point store from an extracted vector, with an integer +// VGETLANE that already exists, store the existing VGETLANEu directly. This can +// help reduce fp register pressure, doesn't require the fp extract and allows +// use of more integer post-inc stores not available with vstr. +static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) { + if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) + return SDValue(); + SDValue Extract = St->getValue(); + EVT VT = Extract.getValueType(); + // For now only uses f16. This may be useful for f32 too, but that will + // be bitcast(extract), not the VGETLANEu we currently check here. + if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) + return SDValue(); + + SDNode *GetLane = + DAG.getNodeIfExists(ARMISD::VGETLANEu, DAG.getVTList(MVT::i32), + {Extract.getOperand(0), Extract.getOperand(1)}); + if (!GetLane) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + SDLoc DL(St); + // Create a new integer store to replace the existing floating point version. + SDValue Ch = St->getChain(); + SDValue BasePtr = St->getBasePtr(); + Align Alignment = St->getOriginalAlign(); + MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags(); + AAMDNodes AAInfo = St->getAAInfo(); + EVT NewToVT = EVT::getIntegerVT(C, VT.getSizeInBits()); + SDValue Store = DAG.getTruncStore(Ch, DL, SDValue(GetLane, 0), BasePtr, + St->getPointerInfo(), NewToVT, + Alignment.value(), MMOFlags, AAInfo); + + return Store; +} + /// PerformSTORECombine - Target-specific dag combine xforms for /// ISD::STORE. static SDValue PerformSTORECombine(SDNode *N, @@ -14765,9 +15783,15 @@ static SDValue PerformSTORECombine(SDNode *N, if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG)) return Store; - if (Subtarget->hasMVEIntegerOps()) + if (Subtarget->hasMVEIntegerOps()) { if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG)) return NewToken; + if (SDValue NewChain = PerformExtractFpToIntStores(St, DCI.DAG)) + return NewChain; + if (SDValue NewToken = + PerformSplittingMVETruncToNarrowingStores(St, DCI.DAG)) + return NewToken; + } if (!ISD::isNormalStore(St)) return SDValue(); @@ -14953,12 +15977,12 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, SDLoc dl(N); // We are looking for something that will have illegal types if left alone, - // but that we can convert to a single instruction undef MVE. For example + // but that we can convert to a single instruction under MVE. For example // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A // or // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B - // Cases: + // The legal cases are: // VADDV u/s 8/16/32 // VMLAV u/s 8/16/32 // VADDLV u/s 32 @@ -14966,6 +15990,13 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, // If the input vector is smaller than legal (v4i8/v4i16 for example) we can // extend it and use v4i32 instead. + auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) { + EVT AVT = A.getValueType(); + return any_of(ExtTypes, [&](MVT Ty) { + return AVT.getVectorNumElements() == Ty.getVectorNumElements() && + AVT.bitsLE(Ty); + }); + }; auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) { EVT AVT = A.getValueType(); if (!AVT.is128BitVector()) @@ -14979,7 +16010,7 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, if (ResVT != RetTy || N0->getOpcode() != ExtendCode) return SDValue(); SDValue A = N0->getOperand(0); - if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; })) + if (ExtTypeMatches(A, ExtTypes)) return ExtendIfNeeded(A, ExtendCode); return SDValue(); }; @@ -14993,7 +16024,7 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, if (Ext->getOpcode() != ExtendCode) return SDValue(); SDValue A = Ext->getOperand(0); - if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; })) + if (ExtTypeMatches(A, ExtTypes)) return ExtendIfNeeded(A, ExtendCode); return SDValue(); }; @@ -15018,13 +16049,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return false; SDValue ExtA = Mul->getOperand(0); SDValue ExtB = Mul->getOperand(1); - if (ExtA->getOpcode() != ExtendCode && ExtB->getOpcode() != ExtendCode) + if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode) return false; A = ExtA->getOperand(0); B = ExtB->getOperand(0); - if (A.getValueType() == B.getValueType() && - llvm::any_of(ExtTypes, - [&A](MVT Ty) { return A.getValueType() == Ty; })) { + if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) { A = ExtendIfNeeded(A, ExtendCode); B = ExtendIfNeeded(B, ExtendCode); return true; @@ -15052,13 +16081,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return false; SDValue ExtA = Mul->getOperand(0); SDValue ExtB = Mul->getOperand(1); - if (ExtA->getOpcode() != ExtendCode && ExtB->getOpcode() != ExtendCode) + if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode) return false; A = ExtA->getOperand(0); B = ExtB->getOperand(0); - if (A.getValueType() == B.getValueType() && - llvm::any_of(ExtTypes, - [&A](MVT Ty) { return A.getValueType() == Ty; })) { + if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) { A = ExtendIfNeeded(A, ExtendCode); B = ExtendIfNeeded(B, ExtendCode); return true; @@ -15066,6 +16093,32 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return false; }; auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) { + // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64 + // reductions. The operands are extended with MVEEXT, but as they are + // reductions the lane orders do not matter. MVEEXT may be combined with + // loads to produce two extending loads, or else they will be expanded to + // VREV/VMOVL. + EVT VT = Ops[0].getValueType(); + if (VT == MVT::v16i8) { + assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) && + "Unexpected illegal long reduction opcode"); + bool IsUnsigned = Opcode == ARMISD::VMLALVu; + + SDValue Ext0 = + DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl, + DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]); + SDValue Ext1 = + DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl, + DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]); + + SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32), + Ext0, Ext1); + SDValue MLA1 = + DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl, + DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1), + Ext0.getValue(1), Ext1.getValue(1)); + return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1)); + } SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops); return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node, SDValue(Node.getNode(), 1)); @@ -15075,11 +16128,9 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A); if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8})) return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A); - if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, - {MVT::v4i8, MVT::v4i16, MVT::v4i32})) + if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32})) return Create64bitNode(ARMISD::VADDLVs, {A}); - if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, - {MVT::v4i8, MVT::v4i16, MVT::v4i32})) + if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32})) return Create64bitNode(ARMISD::VADDLVu, {A}); if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8})) return DAG.getNode(ISD::TRUNCATE, dl, ResVT, @@ -15093,11 +16144,9 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask); if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask)) return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask); - if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, - {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask)) + if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask)) return Create64bitNode(ARMISD::VADDLVps, {A, Mask}); - if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, - {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask)) + if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask)) return Create64bitNode(ARMISD::VADDLVpu, {A, Mask}); if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask)) return DAG.getNode(ISD::TRUNCATE, dl, ResVT, @@ -15111,11 +16160,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B); if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B)) return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B); - if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, - {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B)) + if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32}, + A, B)) return Create64bitNode(ARMISD::VMLALVs, {A, B}); - if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, - {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B)) + if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32}, + A, B)) return Create64bitNode(ARMISD::VMLALVu, {A, B}); if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B)) return DAG.getNode(ISD::TRUNCATE, dl, ResVT, @@ -15124,17 +16173,17 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::TRUNCATE, dl, ResVT, DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B)); - if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask)) + if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, + Mask)) return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask); - if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask)) + if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, + Mask)) return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask); - if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, - {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, - B, Mask)) + if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, + Mask)) return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask}); - if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, - {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, - B, Mask)) + if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, + Mask)) return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask}); if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask)) return DAG.getNode(ISD::TRUNCATE, dl, ResVT, @@ -15171,6 +16220,14 @@ static SDValue PerformVMOVNCombine(SDNode *N, SDValue Op1 = N->getOperand(1); unsigned IsTop = N->getConstantOperandVal(2); + // VMOVNT a undef -> a + // VMOVNB a undef -> a + // VMOVNB undef a -> a + if (Op1->isUndef()) + return Op0; + if (Op0->isUndef() && !IsTop) + return Op1; + // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b) // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b) if ((Op1->getOpcode() == ARMISD::VQMOVNs || @@ -15580,10 +16637,8 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) { EVT FromEltVT = FromVT.getVectorElementType(); unsigned NumElements = 0; - if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8)) + if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8) NumElements = 4; - if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8) - NumElements = 8; if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16) NumElements = 4; if (NumElements == 0 || @@ -15934,7 +16989,7 @@ static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, } case ISD::INTRINSIC_W_CHAIN: { unsigned IntOp = cast<ConstantSDNode>(N.getOperand(1))->getZExtValue(); - if (IntOp != Intrinsic::test_set_loop_iterations && + if (IntOp != Intrinsic::test_start_loop_iterations && IntOp != Intrinsic::loop_decrement_reg) return SDValue(); return N; @@ -15949,7 +17004,7 @@ static SDValue PerformHWLoopCombine(SDNode *N, // The hwloop intrinsics that we're interested are used for control-flow, // either for entering or exiting the loop: - // - test.set.loop.iterations will test whether its operand is zero. If it + // - test.start.loop.iterations will test whether its operand is zero. If it // is zero, the proceeding branch should not enter the loop. // - loop.decrement.reg also tests whether its operand is zero. If it is // zero, the proceeding branch should not branch back to the beginning of @@ -16024,21 +17079,25 @@ static SDValue PerformHWLoopCombine(SDNode *N, DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr); }; - if (IntOp == Intrinsic::test_set_loop_iterations) { + if (IntOp == Intrinsic::test_start_loop_iterations) { SDValue Res; + SDValue Setup = DAG.getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements); // We expect this 'instruction' to branch when the counter is zero. if (IsTrueIfZero(CC, Imm)) { - SDValue Ops[] = { Chain, Elements, Dest }; + SDValue Ops[] = {Chain, Setup, Dest}; Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops); } else { // The logic is the reverse of what we need for WLS, so find the other // basic block target: the target of the proceeding br. UpdateUncondBr(Br, Dest, DAG); - SDValue Ops[] = { Chain, Elements, OtherTarget }; + SDValue Ops[] = {Chain, Setup, OtherTarget}; Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops); } - DAG.ReplaceAllUsesOfValueWith(Int.getValue(1), Int.getOperand(0)); + // Update LR count to the new value + DAG.ReplaceAllUsesOfValueWith(Int.getValue(0), Setup); + // Update chain + DAG.ReplaceAllUsesOfValueWith(Int.getValue(2), Int.getOperand(0)); return Res; } else { SDValue Size = DAG.getTargetConstant( @@ -16283,8 +17342,10 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const { return Res; } -static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG, - const ARMSubtarget *ST) { +static SDValue PerformBITCASTCombine(SDNode *N, + TargetLowering::DAGCombinerInfo &DCI, + const ARMSubtarget *ST) { + SelectionDAG &DAG = DCI.DAG; SDValue Src = N->getOperand(0); EVT DstVT = N->getValueType(0); @@ -16310,9 +17371,288 @@ static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG, DAG.getDataLayout().isBigEndian()) return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src); + // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x + if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI)) + return R; + return SDValue(); } +// Some combines for the MVETrunc truncations legalizer helper. Also lowers the +// node into stack operations after legalizeOps. +SDValue ARMTargetLowering::PerformMVETruncCombine( + SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + EVT VT = N->getValueType(0); + SDLoc DL(N); + + // MVETrunc(Undef, Undef) -> Undef + if (all_of(N->ops(), [](SDValue Op) { return Op.isUndef(); })) + return DAG.getUNDEF(VT); + + // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc + if (N->getNumOperands() == 2 && + N->getOperand(0).getOpcode() == ARMISD::MVETRUNC && + N->getOperand(1).getOpcode() == ARMISD::MVETRUNC) + return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0), + N->getOperand(0).getOperand(1), + N->getOperand(1).getOperand(0), + N->getOperand(1).getOperand(1)); + + // MVETrunc(shuffle, shuffle) -> VMOVN + if (N->getNumOperands() == 2 && + N->getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE && + N->getOperand(1).getOpcode() == ISD::VECTOR_SHUFFLE) { + auto *S0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode()); + auto *S1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode()); + + if (S0->getOperand(0) == S1->getOperand(0) && + S0->getOperand(1) == S1->getOperand(1)) { + // Construct complete shuffle mask + SmallVector<int, 8> Mask(S0->getMask().begin(), S0->getMask().end()); + Mask.append(S1->getMask().begin(), S1->getMask().end()); + + if (isVMOVNTruncMask(Mask, VT, 0)) + return DAG.getNode( + ARMISD::VMOVN, DL, VT, + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), + DAG.getConstant(1, DL, MVT::i32)); + if (isVMOVNTruncMask(Mask, VT, 1)) + return DAG.getNode( + ARMISD::VMOVN, DL, VT, + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), + DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), + DAG.getConstant(1, DL, MVT::i32)); + } + } + + // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the + // truncate to a buildvector to allow the generic optimisations to kick in. + if (all_of(N->ops(), [](SDValue Op) { + return Op.getOpcode() == ISD::BUILD_VECTOR || + Op.getOpcode() == ISD::VECTOR_SHUFFLE || + (Op.getOpcode() == ISD::BITCAST && + Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR); + })) { + SmallVector<SDValue, 8> Extracts; + for (unsigned Op = 0; Op < N->getNumOperands(); Op++) { + SDValue O = N->getOperand(Op); + for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) { + SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, O, + DAG.getConstant(i, DL, MVT::i32)); + Extracts.push_back(Ext); + } + } + return DAG.getBuildVector(VT, DL, Extracts); + } + + // If we are late in the legalization process and nothing has optimised + // the trunc to anything better, lower it to a stack store and reload, + // performing the truncation whilst keeping the lanes in the correct order: + // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack; + if (!DCI.isAfterLegalizeDAG()) + return SDValue(); + + SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::Fixed(16), Align(4)); + int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); + int NumIns = N->getNumOperands(); + assert((NumIns == 2 || NumIns == 4) && + "Expected 2 or 4 inputs to an MVETrunc"); + EVT StoreVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); + if (N->getNumOperands() == 4) + StoreVT = StoreVT.getHalfNumVectorElementsVT(*DAG.getContext()); + + SmallVector<SDValue> Chains; + for (int I = 0; I < NumIns; I++) { + SDValue Ptr = DAG.getNode( + ISD::ADD, DL, StackPtr.getValueType(), StackPtr, + DAG.getConstant(I * 16 / NumIns, DL, StackPtr.getValueType())); + MachinePointerInfo MPI = MachinePointerInfo::getFixedStack( + DAG.getMachineFunction(), SPFI, I * 16 / NumIns); + SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), DL, N->getOperand(I), + Ptr, MPI, StoreVT, Align(4)); + Chains.push_back(Ch); + } + + SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); + MachinePointerInfo MPI = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0); + return DAG.getLoad(VT, DL, Chain, StackPtr, MPI, Align(4)); +} + +// Take a MVEEXT(load x) and split that into (extload x, extload x+8) +static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, + SelectionDAG &DAG) { + SDValue N0 = N->getOperand(0); + LoadSDNode *LD = dyn_cast<LoadSDNode>(N0.getNode()); + if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed()) + return SDValue(); + + EVT FromVT = LD->getMemoryVT(); + EVT ToVT = N->getValueType(0); + if (!ToVT.isVector()) + return SDValue(); + assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2); + EVT ToEltVT = ToVT.getVectorElementType(); + EVT FromEltVT = FromVT.getVectorElementType(); + + unsigned NumElements = 0; + if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8)) + NumElements = 4; + if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8) + NumElements = 8; + assert(NumElements != 0); + + ISD::LoadExtType NewExtType = + N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD; + if (LD->getExtensionType() != ISD::NON_EXTLOAD && + LD->getExtensionType() != ISD::EXTLOAD && + LD->getExtensionType() != NewExtType) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + SDLoc DL(LD); + // Details about the old load + SDValue Ch = LD->getChain(); + SDValue BasePtr = LD->getBasePtr(); + Align Alignment = LD->getOriginalAlign(); + MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); + AAMDNodes AAInfo = LD->getAAInfo(); + + SDValue Offset = DAG.getUNDEF(BasePtr.getValueType()); + EVT NewFromVT = EVT::getVectorVT( + C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements); + EVT NewToVT = EVT::getVectorVT( + C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements); + + SmallVector<SDValue, 4> Loads; + SmallVector<SDValue, 4> Chains; + for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) { + unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8; + SDValue NewPtr = + DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset)); + + SDValue NewLoad = + DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset, + LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT, + Alignment, MMOFlags, AAInfo); + Loads.push_back(NewLoad); + Chains.push_back(SDValue(NewLoad.getNode(), 1)); + } + + SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); + DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain); + return DAG.getMergeValues(Loads, DL); +} + +// Perform combines for MVEEXT. If it has not be optimized to anything better +// before lowering, it gets converted to stack store and extloads performing the +// extend whilst still keeping the same lane ordering. +SDValue ARMTargetLowering::PerformMVEExtCombine( + SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + EVT VT = N->getValueType(0); + SDLoc DL(N); + assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements"); + assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type"); + + EVT ExtVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT( + *DAG.getContext()); + auto Extend = [&](SDValue V) { + SDValue VVT = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, V); + return N->getOpcode() == ARMISD::MVESEXT + ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT, + DAG.getValueType(ExtVT)) + : DAG.getZeroExtendInReg(VVT, DL, ExtVT); + }; + + // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP) + if (N->getOperand(0).getOpcode() == ARMISD::VDUP) { + SDValue Ext = Extend(N->getOperand(0)); + return DAG.getMergeValues({Ext, Ext}, DL); + } + + // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG + if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0))) { + ArrayRef<int> Mask = SVN->getMask(); + assert(Mask.size() == 2 * VT.getVectorNumElements()); + assert(Mask.size() == SVN->getValueType(0).getVectorNumElements()); + unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16; + SDValue Op0 = SVN->getOperand(0); + SDValue Op1 = SVN->getOperand(1); + + auto CheckInregMask = [&](int Start, int Offset) { + for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx) + if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset) + return false; + return true; + }; + SDValue V0 = SDValue(N, 0); + SDValue V1 = SDValue(N, 1); + if (CheckInregMask(0, 0)) + V0 = Extend(Op0); + else if (CheckInregMask(0, 1)) + V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0)); + else if (CheckInregMask(0, Mask.size())) + V0 = Extend(Op1); + else if (CheckInregMask(0, Mask.size() + 1)) + V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1)); + + if (CheckInregMask(VT.getVectorNumElements(), Mask.size())) + V1 = Extend(Op1); + else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1)) + V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1)); + else if (CheckInregMask(VT.getVectorNumElements(), 0)) + V1 = Extend(Op0); + else if (CheckInregMask(VT.getVectorNumElements(), 1)) + V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0)); + + if (V0.getNode() != N || V1.getNode() != N) + return DAG.getMergeValues({V0, V1}, DL); + } + + // MVEEXT(load) -> extload, extload + if (N->getOperand(0)->getOpcode() == ISD::LOAD) + if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG)) + return L; + + if (!DCI.isAfterLegalizeDAG()) + return SDValue(); + + // Lower to a stack store and reload: + // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8; + SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::Fixed(16), Align(4)); + int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); + int NumOuts = N->getNumValues(); + assert((NumOuts == 2 || NumOuts == 4) && + "Expected 2 or 4 outputs to an MVEEXT"); + EVT LoadVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT( + *DAG.getContext()); + if (N->getNumOperands() == 4) + LoadVT = LoadVT.getHalfNumVectorElementsVT(*DAG.getContext()); + + MachinePointerInfo MPI = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0); + SDValue Chain = DAG.getStore(DAG.getEntryNode(), DL, N->getOperand(0), + StackPtr, MPI, Align(4)); + + SmallVector<SDValue> Loads; + for (int I = 0; I < NumOuts; I++) { + SDValue Ptr = DAG.getNode( + ISD::ADD, DL, StackPtr.getValueType(), StackPtr, + DAG.getConstant(I * 16 / NumOuts, DL, StackPtr.getValueType())); + MachinePointerInfo MPI = MachinePointerInfo::getFixedStack( + DAG.getMachineFunction(), SPFI, I * 16 / NumOuts); + SDValue Load = DAG.getExtLoad( + N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL, + VT, Chain, Ptr, MPI, LoadVT, Align(4)); + Loads.push_back(Load); + } + + return DAG.getMergeValues(Loads, DL); +} + SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { switch (N->getOpcode()) { @@ -16334,18 +17674,20 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, case ARMISD::ADDC: case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget); case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget); - case ARMISD::BFI: return PerformBFICombine(N, DCI); + case ARMISD::BFI: return PerformBFICombine(N, DCI.DAG); case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget); case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG); case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI); - case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI); + case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI.DAG); case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget); case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget); case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI); - case ISD::EXTRACT_VECTOR_ELT: return PerformExtractEltCombine(N, DCI); + case ISD::EXTRACT_VECTOR_ELT: + return PerformExtractEltCombine(N, DCI, Subtarget); + case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DCI.DAG); case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG); case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget); - case ARMISD::VDUP: return PerformVDUPCombine(N, DCI, Subtarget); + case ARMISD::VDUP: return PerformVDUPCombine(N, DCI.DAG, Subtarget); case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: return PerformVCVTCombine(N, DCI.DAG, Subtarget); @@ -16379,13 +17721,18 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, case ARMISD::BUILD_VECTOR: return PerformARMBUILD_VECTORCombine(N, DCI); case ISD::BITCAST: - return PerformBITCASTCombine(N, DCI.DAG, Subtarget); + return PerformBITCASTCombine(N, DCI, Subtarget); case ARMISD::PREDICATE_CAST: return PerformPREDICATE_CASTCombine(N, DCI); case ARMISD::VECTOR_REG_CAST: - return PerformVECTOR_REG_CASTCombine(N, DCI, Subtarget); + return PerformVECTOR_REG_CASTCombine(N, DCI.DAG, Subtarget); + case ARMISD::MVETRUNC: + return PerformMVETruncCombine(N, DCI); + case ARMISD::MVESEXT: + case ARMISD::MVEZEXT: + return PerformMVEExtCombine(N, DCI); case ARMISD::VCMP: - return PerformVCMPCombine(N, DCI, Subtarget); + return PerformVCMPCombine(N, DCI.DAG, Subtarget); case ISD::VECREDUCE_ADD: return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget); case ARMISD::VMOVN: @@ -16413,7 +17760,9 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, } case ARMISD::SMLALBB: case ARMISD::QADD16b: - case ARMISD::QSUB16b: { + case ARMISD::QSUB16b: + case ARMISD::UQADD16b: + case ARMISD::UQSUB16b: { unsigned BitWidth = N->getValueType(0).getSizeInBits(); APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16); if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || @@ -16450,7 +17799,9 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N, break; } case ARMISD::QADD8b: - case ARMISD::QSUB8b: { + case ARMISD::QSUB8b: + case ARMISD::UQADD8b: + case ARMISD::UQSUB8b: { unsigned BitWidth = N->getValueType(0).getSizeInBits(); APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 8); if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || @@ -16503,7 +17854,7 @@ bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc, } bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, - unsigned Alignment, + Align Alignment, MachineMemOperand::Flags, bool *Fast) const { // Depends what it gets converted into if the type is weird. @@ -16582,14 +17933,14 @@ EVT ARMTargetLowering::getOptimalMemOpType( bool Fast; if (Op.size() >= 16 && (Op.isAligned(Align(16)) || - (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, + (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, Align(1), MachineMemOperand::MONone, &Fast) && Fast))) { return MVT::v2f64; } else if (Op.size() >= 8 && (Op.isAligned(Align(8)) || (allowsMisalignedMemoryAccesses( - MVT::f64, 0, 1, MachineMemOperand::MONone, &Fast) && + MVT::f64, 0, Align(1), MachineMemOperand::MONone, &Fast) && Fast))) { return MVT::f64; } @@ -16849,9 +18200,10 @@ bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const { return true; } -int ARMTargetLowering::getScalingFactorCost(const DataLayout &DL, - const AddrMode &AM, Type *Ty, - unsigned AS) const { +InstructionCost ARMTargetLowering::getScalingFactorCost(const DataLayout &DL, + const AddrMode &AM, + Type *Ty, + unsigned AS) const { if (isLegalAddressingMode(DL, AM, Ty, AS)) { if (Subtarget->hasFPAO()) return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster @@ -17403,6 +18755,8 @@ bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1)); if (!RHS || RHS->getZExtValue() != 4) return false; + if (Alignment < Align(4)) + return false; Offset = Op->getOperand(1); Base = Op->getOperand(0); @@ -17539,6 +18893,28 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, Known = KnownOp.zext(32); break; } + case ARMISD::CSINC: + case ARMISD::CSINV: + case ARMISD::CSNEG: { + KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1); + KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1); + + // The result is either: + // CSINC: KnownOp0 or KnownOp1 + 1 + // CSINV: KnownOp0 or ~KnownOp1 + // CSNEG: KnownOp0 or KnownOp1 * -1 + if (Op.getOpcode() == ARMISD::CSINC) + KnownOp1 = KnownBits::computeForAddSub( + true, false, KnownOp1, KnownBits::makeConstant(APInt(32, 1))); + else if (Op.getOpcode() == ARMISD::CSINV) + std::swap(KnownOp1.Zero, KnownOp1.One); + else if (Op.getOpcode() == ARMISD::CSNEG) + KnownOp1 = KnownBits::mul( + KnownOp1, KnownBits::makeConstant(APInt(32, -1))); + + Known = KnownBits::commonBits(KnownOp0, KnownOp1); + break; + } } } @@ -17842,7 +19218,7 @@ RCPair ARMTargetLowering::getRegForInlineAsmConstraint( break; } - if (StringRef("{cc}").equals_lower(Constraint)) + if (StringRef("{cc}").equals_insensitive(Constraint)) return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass); return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); @@ -18480,6 +19856,66 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags = MachineMemOperand::MOStore; return true; } + case Intrinsic::arm_mve_vldr_gather_base: + case Intrinsic::arm_mve_vldr_gather_base_predicated: { + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = nullptr; + Info.memVT = MVT::getVT(I.getType()); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + } + case Intrinsic::arm_mve_vldr_gather_base_wb: + case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: { + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = nullptr; + Info.memVT = MVT::getVT(I.getType()->getContainedType(0)); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + } + case Intrinsic::arm_mve_vldr_gather_offset: + case Intrinsic::arm_mve_vldr_gather_offset_predicated: { + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = nullptr; + MVT DataVT = MVT::getVT(I.getType()); + unsigned MemSize = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); + Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize), + DataVT.getVectorNumElements()); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + } + case Intrinsic::arm_mve_vstr_scatter_base: + case Intrinsic::arm_mve_vstr_scatter_base_predicated: { + Info.opc = ISD::INTRINSIC_VOID; + Info.ptrVal = nullptr; + Info.memVT = MVT::getVT(I.getArgOperand(2)->getType()); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOStore; + return true; + } + case Intrinsic::arm_mve_vstr_scatter_base_wb: + case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: { + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = nullptr; + Info.memVT = MVT::getVT(I.getArgOperand(2)->getType()); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOStore; + return true; + } + case Intrinsic::arm_mve_vstr_scatter_offset: + case Intrinsic::arm_mve_vstr_scatter_offset_predicated: { + Info.opc = ISD::INTRINSIC_VOID; + Info.ptrVal = nullptr; + MVT DataVT = MVT::getVT(I.getArgOperand(2)->getType()); + unsigned MemSize = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); + Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize), + DataVT.getVectorNumElements()); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOStore; + return true; + } case Intrinsic::arm_ldaex: case Intrinsic::arm_ldrex: { auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); @@ -18551,7 +19987,7 @@ bool ARMTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, return (Index == 0 || Index == ResVT.getVectorNumElements()); } -Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder, +Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); @@ -18581,7 +20017,7 @@ Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder, } // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html -Instruction *ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder, +Instruction *ARMTargetLowering::emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { switch (Ord) { @@ -18606,7 +20042,7 @@ Instruction *ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder, llvm_unreachable("Unknown fence ordering in emitLeadingFence"); } -Instruction *ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder, +Instruction *ARMTargetLowering::emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { switch (Ord) { @@ -18654,6 +20090,14 @@ ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { if (AI->isFloatingPointOperation()) return AtomicExpansionKind::CmpXChg; + // At -O0, fast-regalloc cannot cope with the live vregs necessary to + // implement atomicrmw without spilling. If the target address is also on the + // stack and close enough to the spill slot, this can lead to a situation + // where the monitor always gets cleared and the atomic operation can never + // succeed. So at -O0 lower this operation to a CAS loop. + if (getTargetMachine().getOptLevel() == CodeGenOpt::None) + return AtomicExpansionKind::CmpXChg; + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); bool hasAtomicRMW = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); return (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) @@ -18661,6 +20105,8 @@ ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { : AtomicExpansionKind::None; } +// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32 +// bits, and up to 64 bits on the non-M profiles. TargetLowering::AtomicExpansionKind ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // At -O0, fast-regalloc cannot cope with the live vregs necessary to @@ -18668,9 +20114,11 @@ ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. + unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits(); bool HasAtomicCmpXchg = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); - if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg) + if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg && + Size <= (Subtarget->isMClass() ? 32U : 64U)) return AtomicExpansionKind::LLSC; return AtomicExpansionKind::None; } @@ -18756,16 +20204,16 @@ bool ARMTargetLowering::shouldExpandShift(SelectionDAG &DAG, SDNode *N) const { return !Subtarget->hasMinSize() || Subtarget->isTargetWindows(); } -Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, +Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, + Value *Addr, AtomicOrdering Ord) const { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); - Type *ValTy = cast<PointerType>(Addr->getType())->getElementType(); bool IsAcquire = isAcquireOrStronger(Ord); // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd // intrinsic must return {i32, i32} and we have to recombine them into a // single i64 here. - if (ValTy->getPrimitiveSizeInBits() == 64) { + if (ValueTy->getPrimitiveSizeInBits() == 64) { Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd; Function *Ldrex = Intrinsic::getDeclaration(M, Int); @@ -18777,31 +20225,29 @@ Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); if (!Subtarget->isLittle()) std::swap (Lo, Hi); - Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); - Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); + Lo = Builder.CreateZExt(Lo, ValueTy, "lo64"); + Hi = Builder.CreateZExt(Hi, ValueTy, "hi64"); return Builder.CreateOr( - Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64"); + Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 32)), "val64"); } Type *Tys[] = { Addr->getType() }; Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex; Function *Ldrex = Intrinsic::getDeclaration(M, Int, Tys); - return Builder.CreateTruncOrBitCast( - Builder.CreateCall(Ldrex, Addr), - cast<PointerType>(Addr->getType())->getElementType()); + return Builder.CreateTruncOrBitCast(Builder.CreateCall(Ldrex, Addr), ValueTy); } void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance( - IRBuilder<> &Builder) const { + IRBuilderBase &Builder) const { if (!Subtarget->hasV7Ops()) return; Module *M = Builder.GetInsertBlock()->getParent()->getParent(); Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::arm_clrex)); } -Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val, - Value *Addr, +Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder, + Value *Val, Value *Addr, AtomicOrdering Ord) const { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); bool IsRelease = isReleaseOrStronger(Ord); @@ -18847,7 +20293,8 @@ ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy, } bool ARMTargetLowering::isLegalInterleavedAccessType( - unsigned Factor, FixedVectorType *VecTy, const DataLayout &DL) const { + unsigned Factor, FixedVectorType *VecTy, Align Alignment, + const DataLayout &DL) const { unsigned VecSize = DL.getTypeSizeInBits(VecTy); unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType()); @@ -18870,6 +20317,9 @@ bool ARMTargetLowering::isLegalInterleavedAccessType( // Ensure the element type is legal. if (ElSize != 8 && ElSize != 16 && ElSize != 32) return false; + // And the alignment if high enough under MVE. + if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8) + return false; // Ensure the total vector size is 64 or a multiple of 128. Types larger than // 128 will be split into multiple interleaved accesses. @@ -18910,11 +20360,12 @@ bool ARMTargetLowering::lowerInterleavedLoad( Type *EltTy = VecTy->getElementType(); const DataLayout &DL = LI->getModule()->getDataLayout(); + Align Alignment = LI->getAlign(); // Skip if we do not have NEON and skip illegal vector types. We can // "legalize" wide vector types into multiple interleaved accesses as long as // the vector types are divisible by 128. - if (!isLegalInterleavedAccessType(Factor, VecTy, DL)) + if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL)) return false; unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL); @@ -19063,11 +20514,12 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI, auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen); const DataLayout &DL = SI->getModule()->getDataLayout(); + Align Alignment = SI->getAlign(); // Skip if we do not have NEON and skip illegal vector types. We can // "legalize" wide vector types into multiple interleaved accesses as long as // the vector types are divisible by 128. - if (!isLegalInterleavedAccessType(Factor, SubVecTy, DL)) + if (!isLegalInterleavedAccessType(Factor, SubVecTy, Alignment, DL)) return false; unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL); @@ -19250,8 +20702,8 @@ static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, } /// Return the correct alignment for the current calling convention. -Align ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy, - DataLayout DL) const { +Align ARMTargetLowering::getABIAlignmentForCallingConv( + Type *ArgTy, const DataLayout &DL) const { const Align ABITypeAlign = DL.getABITypeAlign(ArgTy); if (!ArgTy->isVectorTy()) return ABITypeAlign; @@ -19265,7 +20717,8 @@ Align ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy, /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when /// passing according to AAPCS rules. bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters( - Type *Ty, CallingConv::ID CallConv, bool isVarArg) const { + Type *Ty, CallingConv::ID CallConv, bool isVarArg, + const DataLayout &DL) const { if (getEffectiveCallingConv(CallConv, isVarArg) != CallingConv::ARM_AAPCS_VFP) return false; |