diff options
Diffstat (limited to 'llvm/lib/Target/PowerPC/PPCISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 3635 |
1 files changed, 1451 insertions, 2184 deletions
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index ddfbd04e1ebc..9215c17cb94b 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -74,6 +74,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSectionXCOFF.h" #include "llvm/MC/MCSymbolXCOFF.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/BranchProbability.h" @@ -120,6 +121,11 @@ cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables", cl::desc("use absolute jump tables on ppc"), cl::Hidden); +// TODO - Remove this option if soft fp128 has been fully supported . +static cl::opt<bool> + EnableSoftFP128("enable-soft-fp128", + cl::desc("temp option to enable soft fp128"), cl::Hidden); + STATISTIC(NumTailCalls, "Number of tail calls"); STATISTIC(NumSiblingCalls, "Number of sibling calls"); STATISTIC(ShufflesHandledWithVPERM, "Number of shuffles lowered to a VPERM"); @@ -145,7 +151,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, if (!useSoftFloat()) { if (hasSPE()) { addRegisterClass(MVT::f32, &PPC::GPRCRegClass); - addRegisterClass(MVT::f64, &PPC::SPERCRegClass); + // EFPU2 APU only supports f32 + if (!Subtarget.hasEFPU2()) + addRegisterClass(MVT::f64, &PPC::SPERCRegClass); } else { addRegisterClass(MVT::f32, &PPC::F4RCRegClass); addRegisterClass(MVT::f64, &PPC::F8RCRegClass); @@ -215,13 +223,36 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); if (isPPC64 || Subtarget.hasFPCVT()) { + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Promote); + AddPromotedToType(ISD::STRICT_SINT_TO_FP, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Promote); + AddPromotedToType(ISD::STRICT_UINT_TO_FP, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); + setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); + + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i1, Promote); + AddPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i1, Promote); + AddPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); + + setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote); + AddPromotedToType(ISD::FP_TO_SINT, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); + setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote); + AddPromotedToType(ISD::FP_TO_UINT, MVT::i1, + isPPC64 ? MVT::i64 : MVT::i32); } else { + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); } @@ -247,6 +278,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // PPC (the libcall is not available). setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::ppcf128, Custom); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::ppcf128, Custom); // We do not currently implement these libm ops for PowerPC. setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); @@ -299,8 +332,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal); setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal); - if (Subtarget.hasVSX()) - setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f64, Legal); + if (Subtarget.hasVSX()) { + setOperationAction(ISD::STRICT_FRINT, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FRINT, MVT::f64, Legal); + } if (Subtarget.hasFSQRT()) { setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal); @@ -338,6 +373,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FMA , MVT::f32, Legal); } + if (Subtarget.hasSPE()) + setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); + setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); // If we're enabling GP optimizations, use hardware square root @@ -415,6 +453,16 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, if (!Subtarget.useCRBits()) setOperationAction(ISD::SETCC, MVT::i32, Custom); + if (Subtarget.hasFPU()) { + setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Legal); + + setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Legal); + } + // PowerPC does not have BRCOND which requires SetCC if (!Subtarget.useCRBits()) setOperationAction(ISD::BRCOND, MVT::Other, Expand); @@ -431,9 +479,12 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); } else { // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); // PowerPC does not have [U|S]INT_TO_FP + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Expand); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Expand); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); } @@ -561,36 +612,56 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setCondCodeAction(ISD::SETONE, MVT::f32, Expand); setCondCodeAction(ISD::SETONE, MVT::f64, Expand); + setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal); + if (Subtarget.has64BitSupport()) { // They also have instructions for converting between i64 and fp. + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Expand); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand); setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); // This is just the low 32 bits of a (signed) fp->i64 conversion. // We cannot do this with Promote because i64 is not a legal type. + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); - if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) + if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) { setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); + } } else { // PowerPC does not have FP_TO_UINT on 32-bit implementations. if (Subtarget.hasSPE()) { setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); - } else + } else { + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Expand); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); + } } // With the instructions enabled under FPCVT, we can do everything. if (Subtarget.hasFPCVT()) { if (Subtarget.has64BitSupport()) { + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); } + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom); setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); @@ -613,6 +684,15 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); } + // PowerPC has better expansions for funnel shifts than the generic + // TargetLowering::expandFunnelShift. + if (Subtarget.has64BitSupport()) { + setOperationAction(ISD::FSHL, MVT::i64, Custom); + setOperationAction(ISD::FSHR, MVT::i64, Custom); + } + setOperationAction(ISD::FSHL, MVT::i32, Custom); + setOperationAction(ISD::FSHR, MVT::i32, Custom); + if (Subtarget.hasVSX()) { setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); @@ -745,9 +825,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::UMIN, MVT::v2i64, Expand); } - for (auto VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16, MVT::v16i8}) - setOperationAction(ISD::ABS, VT, Custom); - // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle // with merges, splats, etc. setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); @@ -767,6 +844,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SELECT, MVT::v4i32, Subtarget.useCRBits() ? Legal : Expand); setOperationAction(ISD::STORE , MVT::v4i32, Legal); + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); @@ -776,11 +857,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); - // Without hasP8Altivec set, v2i64 SMAX isn't available. - // But ABS custom lowering requires SMAX support. - if (!Subtarget.hasP8Altivec()) - setOperationAction(ISD::ABS, MVT::v2i64, Expand); - // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8. setOperationAction(ISD::ROTL, MVT::v1i128, Custom); // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w). @@ -799,7 +875,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::MUL, MVT::v4f32, Legal); setOperationAction(ISD::FMA, MVT::v4f32, Legal); - if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { + if (Subtarget.hasVSX()) { setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); } @@ -809,6 +885,27 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, else setOperationAction(ISD::MUL, MVT::v4i32, Custom); + if (Subtarget.isISA3_1()) { + setOperationAction(ISD::MUL, MVT::v2i64, Legal); + setOperationAction(ISD::MULHS, MVT::v2i64, Legal); + setOperationAction(ISD::MULHU, MVT::v2i64, Legal); + setOperationAction(ISD::MULHS, MVT::v4i32, Legal); + setOperationAction(ISD::MULHU, MVT::v4i32, Legal); + setOperationAction(ISD::UDIV, MVT::v2i64, Legal); + setOperationAction(ISD::SDIV, MVT::v2i64, Legal); + setOperationAction(ISD::UDIV, MVT::v4i32, Legal); + setOperationAction(ISD::SDIV, MVT::v4i32, Legal); + setOperationAction(ISD::UREM, MVT::v2i64, Legal); + setOperationAction(ISD::SREM, MVT::v2i64, Legal); + setOperationAction(ISD::UREM, MVT::v4i32, Legal); + setOperationAction(ISD::SREM, MVT::v4i32, Legal); + setOperationAction(ISD::UREM, MVT::v1i128, Legal); + setOperationAction(ISD::SREM, MVT::v1i128, Legal); + setOperationAction(ISD::UDIV, MVT::v1i128, Legal); + setOperationAction(ISD::SDIV, MVT::v1i128, Legal); + setOperationAction(ISD::ROTL, MVT::v1i128, Legal); + } + setOperationAction(ISD::MUL, MVT::v8i16, Legal); setOperationAction(ISD::MUL, MVT::v16i8, Custom); @@ -920,6 +1017,11 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SUB, MVT::v2i64, Expand); } + if (Subtarget.isISA3_1()) + setOperationAction(ISD::SETCC, MVT::v1i128, Legal); + else + setOperationAction(ISD::SETCC, MVT::v1i128, Expand); + setOperationAction(ISD::LOAD, MVT::v2i64, Promote); AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); setOperationAction(ISD::STORE, MVT::v2i64, Promote); @@ -927,6 +1029,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal); + setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal); + setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); @@ -935,6 +1041,14 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // Custom handling for partial vectors of integers converted to // floating point. We already have optimal handling for v2i32 through // the DAG combine, so those aren't necessary. + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i8, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i8, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i16, Custom); + setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i16, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i8, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i8, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i16, Custom); + setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i16, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); @@ -966,7 +1080,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FMAXNUM, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FMINNUM, MVT::v4f32, Legal); - setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v4f32, Legal); + setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FCEIL, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal); @@ -980,7 +1094,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FMAXNUM, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FMINNUM, MVT::v2f64, Legal); - setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v2f64, Legal); + setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FCEIL, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal); @@ -1063,6 +1177,48 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::BSWAP, MVT::v4i32, Legal); setOperationAction(ISD::BSWAP, MVT::v2i64, Legal); setOperationAction(ISD::BSWAP, MVT::v1i128, Legal); + } else if (Subtarget.hasAltivec() && EnableSoftFP128) { + addRegisterClass(MVT::f128, &PPC::VRRCRegClass); + + for (MVT FPT : MVT::fp_valuetypes()) + setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); + + setOperationAction(ISD::LOAD, MVT::f128, Promote); + setOperationAction(ISD::STORE, MVT::f128, Promote); + + AddPromotedToType(ISD::LOAD, MVT::f128, MVT::v4i32); + AddPromotedToType(ISD::STORE, MVT::f128, MVT::v4i32); + + // Set FADD/FSUB as libcall to avoid the legalizer to expand the + // fp_to_uint and int_to_fp. + setOperationAction(ISD::FADD, MVT::f128, LibCall); + setOperationAction(ISD::FSUB, MVT::f128, LibCall); + + setOperationAction(ISD::FMUL, MVT::f128, Expand); + setOperationAction(ISD::FDIV, MVT::f128, Expand); + setOperationAction(ISD::FNEG, MVT::f128, Expand); + setOperationAction(ISD::FABS, MVT::f128, Expand); + setOperationAction(ISD::FSIN, MVT::f128, Expand); + setOperationAction(ISD::FCOS, MVT::f128, Expand); + setOperationAction(ISD::FPOW, MVT::f128, Expand); + setOperationAction(ISD::FPOWI, MVT::f128, Expand); + setOperationAction(ISD::FREM, MVT::f128, Expand); + setOperationAction(ISD::FSQRT, MVT::f128, Expand); + setOperationAction(ISD::FMA, MVT::f128, Expand); + setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); + + setTruncStoreAction(MVT::f128, MVT::f64, Expand); + setTruncStoreAction(MVT::f128, MVT::f32, Expand); + + // Expand the fp_extend if the target type is fp128. + setOperationAction(ISD::FP_EXTEND, MVT::f128, Expand); + setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Expand); + + // Expand the fp_round if the source type is fp128. + for (MVT VT : {MVT::f32, MVT::f64}) { + setOperationAction(ISD::FP_ROUND, VT, Custom); + setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom); + } } if (Subtarget.hasP9Altivec()) { @@ -1079,164 +1235,24 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, } } - if (Subtarget.hasQPX()) { - setOperationAction(ISD::FADD, MVT::v4f64, Legal); - setOperationAction(ISD::FSUB, MVT::v4f64, Legal); - setOperationAction(ISD::FMUL, MVT::v4f64, Legal); - setOperationAction(ISD::FREM, MVT::v4f64, Expand); - - setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); - setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); - - setOperationAction(ISD::LOAD , MVT::v4f64, Custom); - setOperationAction(ISD::STORE , MVT::v4f64, Custom); - - setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); - setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); - - if (!Subtarget.useCRBits()) - setOperationAction(ISD::SELECT, MVT::v4f64, Expand); - setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); - - setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); - setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); - setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); - setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); - setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); - setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); - setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); - - setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); - setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); - - setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); - setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); - - setOperationAction(ISD::FNEG , MVT::v4f64, Legal); - setOperationAction(ISD::FABS , MVT::v4f64, Legal); - setOperationAction(ISD::FSIN , MVT::v4f64, Expand); - setOperationAction(ISD::FCOS , MVT::v4f64, Expand); - setOperationAction(ISD::FPOW , MVT::v4f64, Expand); - setOperationAction(ISD::FLOG , MVT::v4f64, Expand); - setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); - setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); - setOperationAction(ISD::FEXP , MVT::v4f64, Expand); - setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); - - setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); - setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); - - setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); - setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); - - addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); - - setOperationAction(ISD::FADD, MVT::v4f32, Legal); - setOperationAction(ISD::FSUB, MVT::v4f32, Legal); - setOperationAction(ISD::FMUL, MVT::v4f32, Legal); - setOperationAction(ISD::FREM, MVT::v4f32, Expand); - - setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); - setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); - - setOperationAction(ISD::LOAD , MVT::v4f32, Custom); - setOperationAction(ISD::STORE , MVT::v4f32, Custom); - - if (!Subtarget.useCRBits()) - setOperationAction(ISD::SELECT, MVT::v4f32, Expand); - setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); - - setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); - setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); - setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); - setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); - setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); - setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); - setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); - - setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); - setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); - - setOperationAction(ISD::FNEG , MVT::v4f32, Legal); - setOperationAction(ISD::FABS , MVT::v4f32, Legal); - setOperationAction(ISD::FSIN , MVT::v4f32, Expand); - setOperationAction(ISD::FCOS , MVT::v4f32, Expand); - setOperationAction(ISD::FPOW , MVT::v4f32, Expand); - setOperationAction(ISD::FLOG , MVT::v4f32, Expand); - setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); - setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); - setOperationAction(ISD::FEXP , MVT::v4f32, Expand); - setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); - - setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); - setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); - - setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); - setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); - - addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); - - setOperationAction(ISD::AND , MVT::v4i1, Legal); - setOperationAction(ISD::OR , MVT::v4i1, Legal); - setOperationAction(ISD::XOR , MVT::v4i1, Legal); - - if (!Subtarget.useCRBits()) - setOperationAction(ISD::SELECT, MVT::v4i1, Expand); - setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); - - setOperationAction(ISD::LOAD , MVT::v4i1, Custom); - setOperationAction(ISD::STORE , MVT::v4i1, Custom); - - setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); - setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); - setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); - setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); - setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); - setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); - - setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); - - addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); - - setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); - setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); - setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); - setOperationAction(ISD::FROUND, MVT::v4f64, Legal); - - setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); - setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); - setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); - setOperationAction(ISD::FROUND, MVT::v4f32, Legal); - - setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); - - // These need to set FE_INEXACT, and so cannot be vectorized here. - setOperationAction(ISD::FRINT, MVT::v4f64, Expand); - setOperationAction(ISD::FRINT, MVT::v4f32, Expand); - - if (TM.Options.UnsafeFPMath) { - setOperationAction(ISD::FDIV, MVT::v4f64, Legal); - setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); - - setOperationAction(ISD::FDIV, MVT::v4f32, Legal); - setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); - } else { - setOperationAction(ISD::FDIV, MVT::v4f64, Expand); - setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); - - setOperationAction(ISD::FDIV, MVT::v4f32, Expand); - setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); - } - - // TODO: Handle constrained floating-point operations of v4f64 + if (Subtarget.pairedVectorMemops()) { + addRegisterClass(MVT::v256i1, &PPC::VSRpRCRegClass); + setOperationAction(ISD::LOAD, MVT::v256i1, Custom); + setOperationAction(ISD::STORE, MVT::v256i1, Custom); + } + if (Subtarget.hasMMA()) { + addRegisterClass(MVT::v512i1, &PPC::UACCRCRegClass); + setOperationAction(ISD::LOAD, MVT::v512i1, Custom); + setOperationAction(ISD::STORE, MVT::v512i1, Custom); + setOperationAction(ISD::BUILD_VECTOR, MVT::v512i1, Custom); } if (Subtarget.has64BitSupport()) setOperationAction(ISD::PREFETCH, MVT::Other, Legal); + if (Subtarget.isISA3_1()) + setOperationAction(ISD::SRA, MVT::v1i128, Legal); + setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); if (!isPPC64) { @@ -1258,6 +1274,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setLibcallName(RTLIB::SRA_I128, nullptr); } + if (!isPPC64) + setMaxAtomicSizeInBitsSupported(32); + setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); // We have target-specific dag combine patterns for the following nodes: @@ -1295,12 +1314,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setTargetDAGCombine(ISD::SELECT_CC); } - // Use reciprocal estimates. - if (TM.Options.UnsafeFPMath) { - setTargetDAGCombine(ISD::FDIV); - setTargetDAGCombine(ISD::FSQRT); - } - if (Subtarget.hasP9Altivec()) { setTargetDAGCombine(ISD::ABS); setTargetDAGCombine(ISD::VSELECT); @@ -1316,8 +1329,19 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setLibcallName(RTLIB::POW_F128, "powf128"); setLibcallName(RTLIB::FMIN_F128, "fminf128"); setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); - setLibcallName(RTLIB::POWI_F128, "__powikf2"); setLibcallName(RTLIB::REM_F128, "fmodf128"); + setLibcallName(RTLIB::SQRT_F128, "sqrtf128"); + setLibcallName(RTLIB::CEIL_F128, "ceilf128"); + setLibcallName(RTLIB::FLOOR_F128, "floorf128"); + setLibcallName(RTLIB::TRUNC_F128, "truncf128"); + setLibcallName(RTLIB::ROUND_F128, "roundf128"); + setLibcallName(RTLIB::LROUND_F128, "lroundf128"); + setLibcallName(RTLIB::LLROUND_F128, "llroundf128"); + setLibcallName(RTLIB::RINT_F128, "rintf128"); + setLibcallName(RTLIB::LRINT_F128, "lrintf128"); + setLibcallName(RTLIB::LLRINT_F128, "llrintf128"); + setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128"); + setLibcallName(RTLIB::FMA_F128, "fmaf128"); // With 32 condition bits, we don't need to sink (and duplicate) compares // aggressively in CodeGenPrep. @@ -1380,6 +1404,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, MaxLoadsPerMemcmpOptSize = 4; } + IsStrictFPEnabled = true; + // Let the subtarget (CPU) decide if a predictable select is more expensive // than the corresponding branch. This information is used in CGP to decide // when to convert selects into branches. @@ -1422,8 +1448,8 @@ unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, // 16byte and wider vectors are passed on 16byte boundary. // The rest is 8 on PPC64 and 4 on PPC32 boundary. Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4); - if (Subtarget.hasAltivec() || Subtarget.hasQPX()) - getMaxByValAlign(Ty, Alignment, Subtarget.hasQPX() ? Align(32) : Align(16)); + if (Subtarget.hasAltivec()) + getMaxByValAlign(Ty, Alignment, Align(16)); return Alignment.value(); } @@ -1439,16 +1465,6 @@ bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { return VT.isScalarInteger(); } -/// isMulhCheaperThanMulShift - Return true if a mulh[s|u] node for a specific -/// type is cheaper than a multiply followed by a shift. -/// This is true for words and doublewords on 64-bit PowerPC. -bool PPCTargetLowering::isMulhCheaperThanMulShift(EVT Type) const { - if (Subtarget.isPPC64() && (isOperationLegal(ISD::MULHS, Type) || - isOperationLegal(ISD::MULHU, Type))) - return true; - return TargetLowering::isMulhCheaperThanMulShift(Type); -} - const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { switch ((PPCISD::NodeType)Opcode) { case PPCISD::FIRST_NUMBER: break; @@ -1469,6 +1485,10 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { return "PPCISD::FP_TO_SINT_IN_VSR"; case PPCISD::FRE: return "PPCISD::FRE"; case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; + case PPCISD::FTSQRT: + return "PPCISD::FTSQRT"; + case PPCISD::FSQRT: + return "PPCISD::FSQRT"; case PPCISD::STFIWX: return "PPCISD::STFIWX"; case PPCISD::VPERM: return "PPCISD::VPERM"; case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; @@ -1516,7 +1536,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::ANDI_rec_1_GT_BIT: return "PPCISD::ANDI_rec_1_GT_BIT"; case PPCISD::VCMP: return "PPCISD::VCMP"; - case PPCISD::VCMPo: return "PPCISD::VCMPo"; + case PPCISD::VCMP_rec: return "PPCISD::VCMP_rec"; case PPCISD::LBRX: return "PPCISD::LBRX"; case PPCISD::STBRX: return "PPCISD::STBRX"; case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; @@ -1553,6 +1573,8 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; + case PPCISD::PADDI_DTPREL: + return "PPCISD::PADDI_DTPREL"; case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; case PPCISD::SC: return "PPCISD::SC"; case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; @@ -1561,12 +1583,6 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; case PPCISD::VABSD: return "PPCISD::VABSD"; - case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; - case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; - case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; - case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; - case PPCISD::QBFLT: return "PPCISD::QBFLT"; - case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; @@ -1574,8 +1590,35 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; case PPCISD::FP_EXTEND_HALF: return "PPCISD::FP_EXTEND_HALF"; case PPCISD::MAT_PCREL_ADDR: return "PPCISD::MAT_PCREL_ADDR"; + case PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR: + return "PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR"; + case PPCISD::TLS_LOCAL_EXEC_MAT_ADDR: + return "PPCISD::TLS_LOCAL_EXEC_MAT_ADDR"; + case PPCISD::ACC_BUILD: return "PPCISD::ACC_BUILD"; + case PPCISD::PAIR_BUILD: return "PPCISD::PAIR_BUILD"; + case PPCISD::EXTRACT_VSX_REG: return "PPCISD::EXTRACT_VSX_REG"; + case PPCISD::XXMFACC: return "PPCISD::XXMFACC"; case PPCISD::LD_SPLAT: return "PPCISD::LD_SPLAT"; case PPCISD::FNMSUB: return "PPCISD::FNMSUB"; + case PPCISD::STRICT_FADDRTZ: + return "PPCISD::STRICT_FADDRTZ"; + case PPCISD::STRICT_FCTIDZ: + return "PPCISD::STRICT_FCTIDZ"; + case PPCISD::STRICT_FCTIWZ: + return "PPCISD::STRICT_FCTIWZ"; + case PPCISD::STRICT_FCTIDUZ: + return "PPCISD::STRICT_FCTIDUZ"; + case PPCISD::STRICT_FCTIWUZ: + return "PPCISD::STRICT_FCTIWUZ"; + case PPCISD::STRICT_FCFID: + return "PPCISD::STRICT_FCFID"; + case PPCISD::STRICT_FCFIDU: + return "PPCISD::STRICT_FCFIDU"; + case PPCISD::STRICT_FCFIDS: + return "PPCISD::STRICT_FCFIDS"; + case PPCISD::STRICT_FCFIDUS: + return "PPCISD::STRICT_FCFIDUS"; + case PPCISD::LXVRZX: return "PPCISD::LXVRZX"; } return nullptr; } @@ -1585,9 +1628,6 @@ EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, if (!VT.isVector()) return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; - if (Subtarget.hasQPX()) - return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); - return VT.changeVectorElementTypeToInteger(); } @@ -2361,36 +2401,6 @@ SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { return SDValue(); } -/// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift -/// amount, otherwise return -1. -int PPC::isQVALIGNIShuffleMask(SDNode *N) { - EVT VT = N->getValueType(0); - if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) - return -1; - - ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); - - // Find the first non-undef value in the shuffle mask. - unsigned i; - for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) - /*search*/; - - if (i == 4) return -1; // all undef. - - // Otherwise, check to see if the rest of the elements are consecutively - // numbered from this value. - unsigned ShiftAmt = SVOp->getMaskElt(i); - if (ShiftAmt < i) return -1; - ShiftAmt -= i; - - // Check the rest of the elements to see if they are consecutive. - for (++i; i != 4; ++i) - if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) - return -1; - - return ShiftAmt; -} - //===----------------------------------------------------------------------===// // Addressing Mode Selection //===----------------------------------------------------------------------===// @@ -2432,6 +2442,20 @@ bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, return false; } +/// isIntS34Immediate - This method tests if value of node given can be +/// accurately represented as a sign extension from a 34-bit value. If so, +/// this returns true and the immediate. +bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) { + if (!isa<ConstantSDNode>(N)) + return false; + + Imm = (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); + return isInt<34>(Imm); +} +bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) { + return isIntS34Immediate(Op.getNode(), Imm); +} + /// SelectAddressRegReg - Given the specified addressed, check to see if it /// can be represented as an indexed [r+r] operation. Returns false if it /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is @@ -2632,6 +2656,55 @@ bool PPCTargetLowering::SelectAddressRegImm( return true; // [r+0] } +/// Similar to the 16-bit case but for instructions that take a 34-bit +/// displacement field (prefixed loads/stores). +bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp, + SDValue &Base, + SelectionDAG &DAG) const { + // Only on 64-bit targets. + if (N.getValueType() != MVT::i64) + return false; + + SDLoc dl(N); + int64_t Imm = 0; + + if (N.getOpcode() == ISD::ADD) { + if (!isIntS34Immediate(N.getOperand(1), Imm)) + return false; + Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); + if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) + Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); + else + Base = N.getOperand(0); + return true; + } + + if (N.getOpcode() == ISD::OR) { + if (!isIntS34Immediate(N.getOperand(1), Imm)) + return false; + // If this is an or of disjoint bitfields, we can codegen this as an add + // (for better address arithmetic) if the LHS and RHS of the OR are + // provably disjoint. + KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); + if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL) + return false; + if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) + Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); + else + Base = N.getOperand(0); + Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); + return true; + } + + if (isIntS34Immediate(N, Imm)) { // If the address is a 34-bit const. + Disp = DAG.getTargetConstant(Imm, dl, N.getValueType()); + Base = DAG.getRegister(PPC::ZERO8, N.getValueType()); + return true; + } + + return false; +} + /// SelectAddressRegRegOnly - Given the specified addressed, force it to be /// represented as an indexed [r+r] operation. bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, @@ -2761,16 +2834,9 @@ bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, return false; } - // PowerPC doesn't have preinc load/store instructions for vectors (except - // for QPX, which does have preinc r+r forms). - if (VT.isVector()) { - if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { - return false; - } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { - AM = ISD::PRE_INC; - return true; - } - } + // PowerPC doesn't have preinc load/store instructions for vectors + if (VT.isVector()) + return false; if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { // Common code will reject creating a pre-inc form if the base pointer @@ -3065,6 +3131,15 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, TLSModel::Model Model = TM.getTLSModel(GV); if (Model == TLSModel::LocalExec) { + if (Subtarget.isUsingPCRelativeCalls()) { + SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64); + SDValue TGA = DAG.getTargetGlobalAddress( + GV, dl, PtrVT, 0, (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)); + SDValue MatAddr = + DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA); + return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr); + } + SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_HA); SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, @@ -3077,29 +3152,44 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, } if (Model == TLSModel::InitialExec) { - SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); - SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, - PPCII::MO_TLS); - SDValue GOTPtr; - if (is64bit) { - setUsesTOCBasePtr(DAG); - SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); - GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, - PtrVT, GOTReg, TGA); + bool IsPCRel = Subtarget.isUsingPCRelativeCalls(); + SDValue TGA = DAG.getTargetGlobalAddress( + GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0); + SDValue TGATLS = DAG.getTargetGlobalAddress( + GV, dl, PtrVT, 0, + IsPCRel ? (PPCII::MO_TLS | PPCII::MO_PCREL_FLAG) : PPCII::MO_TLS); + SDValue TPOffset; + if (IsPCRel) { + SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA); + TPOffset = DAG.getLoad(MVT::i64, dl, DAG.getEntryNode(), MatPCRel, + MachinePointerInfo()); } else { - if (!TM.isPositionIndependent()) - GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); - else if (picLevel == PICLevel::SmallPIC) - GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); - else - GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); + SDValue GOTPtr; + if (is64bit) { + setUsesTOCBasePtr(DAG); + SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); + GOTPtr = + DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA); + } else { + if (!TM.isPositionIndependent()) + GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); + else if (picLevel == PICLevel::SmallPIC) + GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); + else + GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); + } + TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr); } - SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, - PtrVT, TGA, GOTPtr); return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); } if (Model == TLSModel::GeneralDynamic) { + if (Subtarget.isUsingPCRelativeCalls()) { + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, + PPCII::MO_GOT_TLSGD_PCREL_FLAG); + return DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA); + } + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { @@ -3118,6 +3208,14 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, } if (Model == TLSModel::LocalDynamic) { + if (Subtarget.isUsingPCRelativeCalls()) { + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, + PPCII::MO_GOT_TLSLD_PCREL_FLAG); + SDValue MatPCRel = + DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA); + return DAG.getNode(PPCISD::PADDI_DTPREL, dl, PtrVT, MatPCRel, TGA); + } + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { @@ -3492,11 +3590,6 @@ static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, PPC::F11, PPC::F12, PPC::F13}; -/// QFPR - The set of QPX registers that should be allocated for arguments. -static const MCPhysReg QFPR[] = { - PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, - PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; - /// CalculateStackSlotSize - Calculates the size reserved for this argument on /// the stack. static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, @@ -3526,10 +3619,6 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || ArgVT == MVT::v1i128 || ArgVT == MVT::f128) Alignment = Align(16); - // QPX vector types stored in double-precision are padded to a 32 byte - // boundary. - else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) - Alignment = Align(32); // ByVal parameters are aligned as requested. if (Flags.isByVal()) { @@ -3561,14 +3650,11 @@ static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, /// stack slot (instead of being passed in registers). ArgOffset, /// AvailableFPRs, and AvailableVRs must hold the current argument /// position, and will be updated to account for this argument. -static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, - ISD::ArgFlagsTy Flags, - unsigned PtrByteSize, - unsigned LinkageSize, - unsigned ParamAreaSize, - unsigned &ArgOffset, +static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, + unsigned PtrByteSize, unsigned LinkageSize, + unsigned ParamAreaSize, unsigned &ArgOffset, unsigned &AvailableFPRs, - unsigned &AvailableVRs, bool HasQPX) { + unsigned &AvailableVRs) { bool UseMemory = false; // Respect alignment of argument on the stack. @@ -3592,11 +3678,7 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, // However, if the argument is actually passed in an FPR or a VR, // we don't use memory after all. if (!Flags.isByVal()) { - if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || - // QPX registers overlap with the scalar FP registers. - (HasQPX && (ArgVT == MVT::v4f32 || - ArgVT == MVT::v4f64 || - ArgVT == MVT::v4i1))) + if (ArgVT == MVT::f32 || ArgVT == MVT::f64) if (AvailableFPRs > 0) { --AvailableFPRs; return false; @@ -3631,11 +3713,8 @@ SDValue PPCTargetLowering::LowerFormalArguments( if (Subtarget.is64BitELFABI()) return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); - if (Subtarget.is32BitELFABI()) - return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, - InVals); - - return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, dl, DAG, + assert(Subtarget.is32BitELFABI()); + return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); } @@ -3735,18 +3814,12 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( RC = &PPC::VRRCRegClass; break; case MVT::v4f32: - RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; + RC = &PPC::VRRCRegClass; break; case MVT::v2f64: case MVT::v2i64: RC = &PPC::VRRCRegClass; break; - case MVT::v4f64: - RC = &PPC::QFRCRegClass; - break; - case MVT::v4i1: - RC = &PPC::QBRCRegClass; - break; } SDValue ArgValue; @@ -3945,7 +4018,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( const unsigned Num_GPR_Regs = array_lengthof(GPR); const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; const unsigned Num_VR_Regs = array_lengthof(VR); - const unsigned Num_QFPR_Regs = Num_FPR_Regs; // Do a first pass over the arguments to determine whether the ABI // guarantees that our caller has allocated the parameter save area @@ -3964,8 +4036,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, PtrByteSize, LinkageSize, ParamAreaSize, - NumBytes, AvailableFPRs, AvailableVRs, - Subtarget.hasQPX())) + NumBytes, AvailableFPRs, AvailableVRs)) HasParameterArea = true; } @@ -3975,7 +4046,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( unsigned ArgOffset = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; - unsigned &QFPR_idx = FPR_idx; SmallVector<SDValue, 8> MemOps; Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); unsigned CurArgIdx = 0; @@ -4218,51 +4288,20 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( case MVT::v2i64: case MVT::v1i128: case MVT::f128: - if (!Subtarget.hasQPX()) { - // These can be scalar arguments or elements of a vector array type - // passed directly. The latter are used to implement ELFv2 homogenous - // vector aggregates. - if (VR_idx != Num_VR_Regs) { - unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); - ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); - ++VR_idx; - } else { - if (CallConv == CallingConv::Fast) - ComputeArgOffset(); - needsLoad = true; - } - if (CallConv != CallingConv::Fast || needsLoad) - ArgOffset += 16; - break; - } // not QPX - - assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && - "Invalid QPX parameter type"); - LLVM_FALLTHROUGH; - - case MVT::v4f64: - case MVT::v4i1: - // QPX vectors are treated like their scalar floating-point subregisters - // (except that they're larger). - unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; - if (QFPR_idx != Num_QFPR_Regs) { - const TargetRegisterClass *RC; - switch (ObjectVT.getSimpleVT().SimpleTy) { - case MVT::v4f64: RC = &PPC::QFRCRegClass; break; - case MVT::v4f32: RC = &PPC::QSRCRegClass; break; - default: RC = &PPC::QBRCRegClass; break; - } - - unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); + // These can be scalar arguments or elements of a vector array type + // passed directly. The latter are used to implement ELFv2 homogenous + // vector aggregates. + if (VR_idx != Num_VR_Regs) { + unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); - ++QFPR_idx; + ++VR_idx; } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } if (CallConv != CallingConv::Fast || needsLoad) - ArgOffset += Sz; + ArgOffset += 16; break; } @@ -4329,366 +4368,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( return Chain; } -SDValue PPCTargetLowering::LowerFormalArguments_Darwin( - SDValue Chain, CallingConv::ID CallConv, bool isVarArg, - const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, - SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { - // TODO: add description of PPC stack frame format, or at least some docs. - // - MachineFunction &MF = DAG.getMachineFunction(); - MachineFrameInfo &MFI = MF.getFrameInfo(); - PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); - - EVT PtrVT = getPointerTy(MF.getDataLayout()); - bool isPPC64 = PtrVT == MVT::i64; - // Potential tail calls could cause overwriting of argument stack slots. - bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && - (CallConv == CallingConv::Fast)); - unsigned PtrByteSize = isPPC64 ? 8 : 4; - unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); - unsigned ArgOffset = LinkageSize; - // Area that is at least reserved in caller of this function. - unsigned MinReservedArea = ArgOffset; - - static const MCPhysReg GPR_32[] = { // 32-bit registers. - PPC::R3, PPC::R4, PPC::R5, PPC::R6, - PPC::R7, PPC::R8, PPC::R9, PPC::R10, - }; - static const MCPhysReg GPR_64[] = { // 64-bit registers. - PPC::X3, PPC::X4, PPC::X5, PPC::X6, - PPC::X7, PPC::X8, PPC::X9, PPC::X10, - }; - static const MCPhysReg VR[] = { - PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, - PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 - }; - - const unsigned Num_GPR_Regs = array_lengthof(GPR_32); - const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; - const unsigned Num_VR_Regs = array_lengthof( VR); - - unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; - - const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; - - // In 32-bit non-varargs functions, the stack space for vectors is after the - // stack space for non-vectors. We do not use this space unless we have - // too many vectors to fit in registers, something that only occurs in - // constructed examples:), but we have to walk the arglist to figure - // that out...for the pathological case, compute VecArgOffset as the - // start of the vector parameter area. Computing VecArgOffset is the - // entire point of the following loop. - unsigned VecArgOffset = ArgOffset; - if (!isVarArg && !isPPC64) { - for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; - ++ArgNo) { - EVT ObjectVT = Ins[ArgNo].VT; - ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; - - if (Flags.isByVal()) { - // ObjSize is the true size, ArgSize rounded up to multiple of regs. - unsigned ObjSize = Flags.getByValSize(); - unsigned ArgSize = - ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; - VecArgOffset += ArgSize; - continue; - } - - switch(ObjectVT.getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unhandled argument type!"); - case MVT::i1: - case MVT::i32: - case MVT::f32: - VecArgOffset += 4; - break; - case MVT::i64: // PPC64 - case MVT::f64: - // FIXME: We are guaranteed to be !isPPC64 at this point. - // Does MVT::i64 apply? - VecArgOffset += 8; - break; - case MVT::v4f32: - case MVT::v4i32: - case MVT::v8i16: - case MVT::v16i8: - // Nothing to do, we're only looking at Nonvector args here. - break; - } - } - } - // We've found where the vector parameter area in memory is. Skip the - // first 12 parameters; these don't use that memory. - VecArgOffset = ((VecArgOffset+15)/16)*16; - VecArgOffset += 12*16; - - // Add DAG nodes to load the arguments or copy them out of registers. On - // entry to a function on PPC, the arguments start after the linkage area, - // although the first ones are often in registers. - - SmallVector<SDValue, 8> MemOps; - unsigned nAltivecParamsAtEnd = 0; - Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); - unsigned CurArgIdx = 0; - for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { - SDValue ArgVal; - bool needsLoad = false; - EVT ObjectVT = Ins[ArgNo].VT; - unsigned ObjSize = ObjectVT.getSizeInBits()/8; - unsigned ArgSize = ObjSize; - ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; - if (Ins[ArgNo].isOrigArg()) { - std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); - CurArgIdx = Ins[ArgNo].getOrigArgIndex(); - } - unsigned CurArgOffset = ArgOffset; - - // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. - if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || - ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { - if (isVarArg || isPPC64) { - MinReservedArea = ((MinReservedArea+15)/16)*16; - MinReservedArea += CalculateStackSlotSize(ObjectVT, - Flags, - PtrByteSize); - } else nAltivecParamsAtEnd++; - } else - // Calculate min reserved area. - MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, - Flags, - PtrByteSize); - - // FIXME the codegen can be much improved in some cases. - // We do not have to keep everything in memory. - if (Flags.isByVal()) { - assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); - - // ObjSize is the true size, ArgSize rounded up to multiple of registers. - ObjSize = Flags.getByValSize(); - ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; - // Objects of size 1 and 2 are right justified, everything else is - // left justified. This means the memory address is adjusted forwards. - if (ObjSize==1 || ObjSize==2) { - CurArgOffset = CurArgOffset + (4 - ObjSize); - } - // The value of the object is its address. - int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, false, true); - SDValue FIN = DAG.getFrameIndex(FI, PtrVT); - InVals.push_back(FIN); - if (ObjSize==1 || ObjSize==2) { - if (GPR_idx != Num_GPR_Regs) { - unsigned VReg; - if (isPPC64) - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); - else - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); - SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); - EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; - SDValue Store = - DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, - MachinePointerInfo(&*FuncArg), ObjType); - MemOps.push_back(Store); - ++GPR_idx; - } - - ArgOffset += PtrByteSize; - - continue; - } - for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { - // Store whatever pieces of the object are in registers - // to memory. ArgOffset will be the address of the beginning - // of the object. - if (GPR_idx != Num_GPR_Regs) { - unsigned VReg; - if (isPPC64) - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); - else - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); - int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); - SDValue FIN = DAG.getFrameIndex(FI, PtrVT); - SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); - SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, - MachinePointerInfo(&*FuncArg, j)); - MemOps.push_back(Store); - ++GPR_idx; - ArgOffset += PtrByteSize; - } else { - ArgOffset += ArgSize - (ArgOffset-CurArgOffset); - break; - } - } - continue; - } - - switch (ObjectVT.getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unhandled argument type!"); - case MVT::i1: - case MVT::i32: - if (!isPPC64) { - if (GPR_idx != Num_GPR_Regs) { - unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); - ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); - - if (ObjectVT == MVT::i1) - ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); - - ++GPR_idx; - } else { - needsLoad = true; - ArgSize = PtrByteSize; - } - // All int arguments reserve stack space in the Darwin ABI. - ArgOffset += PtrByteSize; - break; - } - LLVM_FALLTHROUGH; - case MVT::i64: // PPC64 - if (GPR_idx != Num_GPR_Regs) { - unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); - ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); - - if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) - // PPC64 passes i8, i16, and i32 values in i64 registers. Promote - // value to MVT::i64 and then truncate to the correct register size. - ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); - - ++GPR_idx; - } else { - needsLoad = true; - ArgSize = PtrByteSize; - } - // All int arguments reserve stack space in the Darwin ABI. - ArgOffset += 8; - break; - - case MVT::f32: - case MVT::f64: - // Every 4 bytes of argument space consumes one of the GPRs available for - // argument passing. - if (GPR_idx != Num_GPR_Regs) { - ++GPR_idx; - if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) - ++GPR_idx; - } - if (FPR_idx != Num_FPR_Regs) { - unsigned VReg; - - if (ObjectVT == MVT::f32) - VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); - else - VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); - - ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); - ++FPR_idx; - } else { - needsLoad = true; - } - - // All FP arguments reserve stack space in the Darwin ABI. - ArgOffset += isPPC64 ? 8 : ObjSize; - break; - case MVT::v4f32: - case MVT::v4i32: - case MVT::v8i16: - case MVT::v16i8: - // Note that vector arguments in registers don't reserve stack space, - // except in varargs functions. - if (VR_idx != Num_VR_Regs) { - unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); - ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); - if (isVarArg) { - while ((ArgOffset % 16) != 0) { - ArgOffset += PtrByteSize; - if (GPR_idx != Num_GPR_Regs) - GPR_idx++; - } - ArgOffset += 16; - GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? - } - ++VR_idx; - } else { - if (!isVarArg && !isPPC64) { - // Vectors go after all the nonvectors. - CurArgOffset = VecArgOffset; - VecArgOffset += 16; - } else { - // Vectors are aligned. - ArgOffset = ((ArgOffset+15)/16)*16; - CurArgOffset = ArgOffset; - ArgOffset += 16; - } - needsLoad = true; - } - break; - } - - // We need to load the argument to a virtual register if we determined above - // that we ran out of physical registers of the appropriate type. - if (needsLoad) { - int FI = MFI.CreateFixedObject(ObjSize, - CurArgOffset + (ArgSize - ObjSize), - isImmutable); - SDValue FIN = DAG.getFrameIndex(FI, PtrVT); - ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); - } - - InVals.push_back(ArgVal); - } - - // Allow for Altivec parameters at the end, if needed. - if (nAltivecParamsAtEnd) { - MinReservedArea = ((MinReservedArea+15)/16)*16; - MinReservedArea += 16*nAltivecParamsAtEnd; - } - - // Area that is at least reserved in the caller of this function. - MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); - - // Set the size that is at least reserved in caller of this function. Tail - // call optimized functions' reserved stack space needs to be aligned so that - // taking the difference between two stack areas will result in an aligned - // stack. - MinReservedArea = - EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); - FuncInfo->setMinReservedArea(MinReservedArea); - - // If the function takes variable number of arguments, make a frame index for - // the start of the first vararg value... for expansion of llvm.va_start. - if (isVarArg) { - int Depth = ArgOffset; - - FuncInfo->setVarArgsFrameIndex( - MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, - Depth, true)); - SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); - - // If this function is vararg, store any remaining integer argument regs - // to their spots on the stack so that they may be loaded by dereferencing - // the result of va_next. - for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { - unsigned VReg; - - if (isPPC64) - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); - else - VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); - - SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); - SDValue Store = - DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); - MemOps.push_back(Store); - // Increment the address by four for the next argument to store - SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); - FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); - } - } - - if (!MemOps.empty()) - Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); - - return Chain; -} - /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be /// adjusted to accommodate the arguments for the tailcall. static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, @@ -4759,6 +4438,13 @@ static bool callsShareTOCBase(const Function *Caller, SDValue Callee, if (STICallee->isUsingPCRelativeCalls()) return false; + // If the GV is not a strong definition then we need to assume it can be + // replaced by another function at link time. The function that replaces + // it may not share the same TOC as the caller since the callee may be + // replaced by a PC Relative version of the same function. + if (!GV->isStrongDefinitionForLinker()) + return false; + // The medium and large code models are expected to provide a sufficiently // large TOC to provide all data addressing needs of a module with a // single TOC. @@ -4766,12 +4452,6 @@ static bool callsShareTOCBase(const Function *Caller, SDValue Callee, CodeModel::Large == TM.getCodeModel()) return true; - // Otherwise we need to ensure callee and caller are in the same section, - // since the linker may allocate multiple TOCs, and we don't know which - // sections will belong to the same TOC base. - if (!GV->isStrongDefinitionForLinker()) - return false; - // Any explicitly-specified sections and section prefixes must also match. // Also, if we're using -ffunction-sections, then each function is always in // a different section (the same is true for COMDAT functions). @@ -4815,10 +4495,9 @@ needStackSlotPassParameters(const PPCSubtarget &Subtarget, for (const ISD::OutputArg& Param : Outs) { if (Param.Flags.isNest()) continue; - if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, - PtrByteSize, LinkageSize, ParamAreaSize, - NumBytes, AvailableFPRs, AvailableVRs, - Subtarget.hasQPX())) + if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize, + LinkageSize, ParamAreaSize, NumBytes, + AvailableFPRs, AvailableVRs)) return true; } return false; @@ -5332,66 +5011,53 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG, Subtarget.is32BitELFABI() && !isLocalCallee() && Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; - // On AIX, direct function calls reference the symbol for the function's - // entry point, which is named by prepending a "." before the function's - // C-linkage name. - const auto getAIXFuncEntryPointSymbolSDNode = - [&](StringRef FuncName, bool IsDeclaration, - const XCOFF::StorageClass &SC) { - auto &Context = DAG.getMachineFunction().getMMI().getContext(); - - MCSymbolXCOFF *S = cast<MCSymbolXCOFF>( - Context.getOrCreateSymbol(Twine(".") + Twine(FuncName))); + const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) { + const TargetMachine &TM = Subtarget.getTargetMachine(); + const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering(); + MCSymbolXCOFF *S = + cast<MCSymbolXCOFF>(TLOF->getFunctionEntryPointSymbol(GV, TM)); - if (IsDeclaration && !S->hasRepresentedCsectSet()) { - // On AIX, an undefined symbol needs to be associated with a - // MCSectionXCOFF to get the correct storage mapping class. - // In this case, XCOFF::XMC_PR. - MCSectionXCOFF *Sec = Context.getXCOFFSection( - S->getSymbolTableName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC, - SectionKind::getMetadata()); - S->setRepresentedCsect(Sec); - } - - MVT PtrVT = - DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); - return DAG.getMCSymbol(S, PtrVT); - }; + MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); + return DAG.getMCSymbol(S, PtrVT); + }; if (isFunctionGlobalAddress(Callee)) { - const GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee); - const GlobalValue *GV = G->getGlobal(); + const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal(); - if (!Subtarget.isAIXABI()) - return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0, - UsePlt ? PPCII::MO_PLT : 0); - - assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX."); - const GlobalObject *GO = cast<GlobalObject>(GV); - const XCOFF::StorageClass SC = - TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GO); - return getAIXFuncEntryPointSymbolSDNode(GO->getName(), GO->isDeclaration(), - SC); + if (Subtarget.isAIXABI()) { + assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX."); + return getAIXFuncEntryPointSymbolSDNode(GV); + } + return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0, + UsePlt ? PPCII::MO_PLT : 0); } if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { const char *SymName = S->getSymbol(); - if (!Subtarget.isAIXABI()) - return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(), - UsePlt ? PPCII::MO_PLT : 0); + if (Subtarget.isAIXABI()) { + // If there exists a user-declared function whose name is the same as the + // ExternalSymbol's, then we pick up the user-declared version. + const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); + if (const Function *F = + dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) + return getAIXFuncEntryPointSymbolSDNode(F); - // If there exists a user-declared function whose name is the same as the - // ExternalSymbol's, then we pick up the user-declared version. - const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); - if (const Function *F = - dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) { - const XCOFF::StorageClass SC = - TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F); - return getAIXFuncEntryPointSymbolSDNode(F->getName(), F->isDeclaration(), - SC); - } + // On AIX, direct function calls reference the symbol for the function's + // entry point, which is named by prepending a "." before the function's + // C-linkage name. A Qualname is returned here because an external + // function entry point is a csect with XTY_ER property. + const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) { + auto &Context = DAG.getMachineFunction().getMMI().getContext(); + MCSectionXCOFF *Sec = Context.getXCOFFSection( + (Twine(".") + Twine(SymName)).str(), XCOFF::XMC_PR, XCOFF::XTY_ER, + SectionKind::getMetadata()); + return Sec->getQualNameSymbol(); + }; - return getAIXFuncEntryPointSymbolSDNode(SymName, true, XCOFF::C_EXT); + SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data(); + } + return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(), + UsePlt ? PPCII::MO_PLT : 0); } // No transformation needed. @@ -5736,19 +5402,15 @@ PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }), CLI.NoMerge); - if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) - return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, - InVals, CB); - - if (Subtarget.isSVR4ABI()) - return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, - InVals, CB); - if (Subtarget.isAIXABI()) return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CB); - return LowerCall_Darwin(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, + assert(Subtarget.isSVR4ABI()); + if (Subtarget.isPPC64()) + return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, + InVals, CB); + return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CB); } @@ -6045,7 +5707,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned NumBytes = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; - unsigned &QFPR_idx = FPR_idx; static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, @@ -6059,7 +5720,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( const unsigned NumGPRs = array_lengthof(GPR); const unsigned NumFPRs = useSoftFloat() ? 0 : 13; const unsigned NumVRs = array_lengthof(VR); - const unsigned NumQFPRs = NumFPRs; // On ELFv2, we can avoid allocating the parameter area if all the arguments // can be passed to the callee in registers. @@ -6074,9 +5734,8 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( for (unsigned i = 0; i != NumOps; ++i) { if (Outs[i].Flags.isNest()) continue; if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, - PtrByteSize, LinkageSize, ParamAreaSize, - NumBytesTmp, AvailableFPRs, AvailableVRs, - Subtarget.hasQPX())) + PtrByteSize, LinkageSize, ParamAreaSize, + NumBytesTmp, AvailableFPRs, AvailableVRs)) HasParameterArea = true; } } @@ -6124,20 +5783,11 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( continue; break; case MVT::v4f32: - // When using QPX, this is handled like a FP register, otherwise, it - // is an Altivec register. - if (Subtarget.hasQPX()) { - if (++NumFPRsUsed <= NumFPRs) - continue; - } else { - if (++NumVRsUsed <= NumVRs) - continue; - } + if (++NumVRsUsed <= NumVRs) + continue; break; case MVT::f32: case MVT::f64: - case MVT::v4f64: // QPX - case MVT::v4i1: // QPX if (++NumFPRsUsed <= NumFPRs) continue; break; @@ -6499,7 +6149,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( case MVT::v2i64: case MVT::v1i128: case MVT::f128: - if (!Subtarget.hasQPX()) { // These can be scalar arguments or elements of a vector array type // passed directly. The latter are used to implement ELFv2 homogenous // vector aggregates. @@ -6555,63 +6204,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( if (!IsFastCall) ArgOffset += 16; break; - } // not QPX - - assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && - "Invalid QPX parameter type"); - - LLVM_FALLTHROUGH; - case MVT::v4f64: - case MVT::v4i1: { - bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; - if (CFlags.IsVarArg) { - assert(HasParameterArea && - "Parameter area must exist if we have a varargs call."); - // We could elide this store in the case where the object fits - // entirely in R registers. Maybe later. - SDValue Store = - DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Store); - if (QFPR_idx != NumQFPRs) { - SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store, - PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); - } - ArgOffset += (IsF32 ? 16 : 32); - for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { - if (GPR_idx == NumGPRs) - break; - SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, - DAG.getConstant(i, dl, PtrVT)); - SDValue Load = - DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - } - break; - } - - // Non-varargs QPX params go into registers or on the stack. - if (QFPR_idx != NumQFPRs) { - RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); - } else { - if (IsFastCall) - ComputePtrOff(); - - assert(HasParameterArea && - "Parameter area must exist to pass an argument in memory."); - LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, - true, CFlags.IsTailCall, true, MemOpChains, - TailCallArguments, dl); - if (IsFastCall) - ArgOffset += (IsF32 ? 16 : 32); - } - - if (!IsFastCall) - ArgOffset += (IsF32 ? 16 : 32); - break; - } } } @@ -6665,384 +6257,6 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( Callee, SPDiff, NumBytes, Ins, InVals, CB); } -SDValue PPCTargetLowering::LowerCall_Darwin( - SDValue Chain, SDValue Callee, CallFlags CFlags, - const SmallVectorImpl<ISD::OutputArg> &Outs, - const SmallVectorImpl<SDValue> &OutVals, - const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, - SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, - const CallBase *CB) const { - unsigned NumOps = Outs.size(); - - EVT PtrVT = getPointerTy(DAG.getDataLayout()); - bool isPPC64 = PtrVT == MVT::i64; - unsigned PtrByteSize = isPPC64 ? 8 : 4; - - MachineFunction &MF = DAG.getMachineFunction(); - - // Mark this function as potentially containing a function that contains a - // tail call. As a consequence the frame pointer will be used for dynamicalloc - // and restoring the callers stack pointer in this functions epilog. This is - // done because by tail calling the called function might overwrite the value - // in this function's (MF) stack pointer stack slot 0(SP). - if (getTargetMachine().Options.GuaranteedTailCallOpt && - CFlags.CallConv == CallingConv::Fast) - MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); - - // Count how many bytes are to be pushed on the stack, including the linkage - // area, and parameter passing area. We start with 24/48 bytes, which is - // prereserved space for [SP][CR][LR][3 x unused]. - unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); - unsigned NumBytes = LinkageSize; - - // Add up all the space actually used. - // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually - // they all go in registers, but we must reserve stack space for them for - // possible use by the caller. In varargs or 64-bit calls, parameters are - // assigned stack space in order, with padding so Altivec parameters are - // 16-byte aligned. - unsigned nAltivecParamsAtEnd = 0; - for (unsigned i = 0; i != NumOps; ++i) { - ISD::ArgFlagsTy Flags = Outs[i].Flags; - EVT ArgVT = Outs[i].VT; - // Varargs Altivec parameters are padded to a 16 byte boundary. - if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || - ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || - ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { - if (!CFlags.IsVarArg && !isPPC64) { - // Non-varargs Altivec parameters go after all the non-Altivec - // parameters; handle those later so we know how much padding we need. - nAltivecParamsAtEnd++; - continue; - } - // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. - NumBytes = ((NumBytes+15)/16)*16; - } - NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); - } - - // Allow for Altivec parameters at the end, if needed. - if (nAltivecParamsAtEnd) { - NumBytes = ((NumBytes+15)/16)*16; - NumBytes += 16*nAltivecParamsAtEnd; - } - - // The prolog code of the callee may store up to 8 GPR argument registers to - // the stack, allowing va_start to index over them in memory if its varargs. - // Because we cannot tell if this is needed on the caller side, we have to - // conservatively assume that it is needed. As such, make sure we have at - // least enough stack space for the caller to store the 8 GPRs. - NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); - - // Tail call needs the stack to be aligned. - if (getTargetMachine().Options.GuaranteedTailCallOpt && - CFlags.CallConv == CallingConv::Fast) - NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); - - // Calculate by how many bytes the stack has to be adjusted in case of tail - // call optimization. - int SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes); - - // To protect arguments on the stack from being clobbered in a tail call, - // force all the loads to happen before doing any other lowering. - if (CFlags.IsTailCall) - Chain = DAG.getStackArgumentTokenFactor(Chain); - - // Adjust the stack pointer for the new arguments... - // These operations are automatically eliminated by the prolog/epilog pass - Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); - SDValue CallSeqStart = Chain; - - // Load the return address and frame pointer so it can be move somewhere else - // later. - SDValue LROp, FPOp; - Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); - - // Set up a copy of the stack pointer for use loading and storing any - // arguments that may not fit in the registers available for argument - // passing. - SDValue StackPtr; - if (isPPC64) - StackPtr = DAG.getRegister(PPC::X1, MVT::i64); - else - StackPtr = DAG.getRegister(PPC::R1, MVT::i32); - - // Figure out which arguments are going to go in registers, and which in - // memory. Also, if this is a vararg function, floating point operations - // must be stored to our stack, and loaded into integer regs as well, if - // any integer regs are available for argument passing. - unsigned ArgOffset = LinkageSize; - unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; - - static const MCPhysReg GPR_32[] = { // 32-bit registers. - PPC::R3, PPC::R4, PPC::R5, PPC::R6, - PPC::R7, PPC::R8, PPC::R9, PPC::R10, - }; - static const MCPhysReg GPR_64[] = { // 64-bit registers. - PPC::X3, PPC::X4, PPC::X5, PPC::X6, - PPC::X7, PPC::X8, PPC::X9, PPC::X10, - }; - static const MCPhysReg VR[] = { - PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, - PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 - }; - const unsigned NumGPRs = array_lengthof(GPR_32); - const unsigned NumFPRs = 13; - const unsigned NumVRs = array_lengthof(VR); - - const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; - - SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; - SmallVector<TailCallArgumentInfo, 8> TailCallArguments; - - SmallVector<SDValue, 8> MemOpChains; - for (unsigned i = 0; i != NumOps; ++i) { - SDValue Arg = OutVals[i]; - ISD::ArgFlagsTy Flags = Outs[i].Flags; - - // PtrOff will be used to store the current argument to the stack if a - // register cannot be found for it. - SDValue PtrOff; - - PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); - - PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); - - // On PPC64, promote integers to 64-bit values. - if (isPPC64 && Arg.getValueType() == MVT::i32) { - // FIXME: Should this use ANY_EXTEND if neither sext nor zext? - unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; - Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); - } - - // FIXME memcpy is used way more than necessary. Correctness first. - // Note: "by value" is code for passing a structure by value, not - // basic types. - if (Flags.isByVal()) { - unsigned Size = Flags.getByValSize(); - // Very small objects are passed right-justified. Everything else is - // passed left-justified. - if (Size==1 || Size==2) { - EVT VT = (Size==1) ? MVT::i8 : MVT::i16; - if (GPR_idx != NumGPRs) { - SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, - MachinePointerInfo(), VT); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - - ArgOffset += PtrByteSize; - } else { - SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, - PtrOff.getValueType()); - SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); - Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, - CallSeqStart, - Flags, DAG, dl); - ArgOffset += PtrByteSize; - } - continue; - } - // Copy entire object into memory. There are cases where gcc-generated - // code assumes it is there, even if it could be put entirely into - // registers. (This is not what the doc says.) - Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, - CallSeqStart, - Flags, DAG, dl); - - // For small aggregates (Darwin only) and aggregates >= PtrByteSize, - // copy the pieces of the object that fit into registers from the - // parameter save area. - for (unsigned j=0; j<Size; j+=PtrByteSize) { - SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); - SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); - if (GPR_idx != NumGPRs) { - SDValue Load = - DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - ArgOffset += PtrByteSize; - } else { - ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; - break; - } - } - continue; - } - - switch (Arg.getSimpleValueType().SimpleTy) { - default: llvm_unreachable("Unexpected ValueType for argument!"); - case MVT::i1: - case MVT::i32: - case MVT::i64: - if (GPR_idx != NumGPRs) { - if (Arg.getValueType() == MVT::i1) - Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg); - - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); - } else { - LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, - isPPC64, CFlags.IsTailCall, false, MemOpChains, - TailCallArguments, dl); - } - ArgOffset += PtrByteSize; - break; - case MVT::f32: - case MVT::f64: - if (FPR_idx != NumFPRs) { - RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); - - if (CFlags.IsVarArg) { - SDValue Store = - DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Store); - - // Float varargs are always shadowed in available integer registers - if (GPR_idx != NumGPRs) { - SDValue Load = - DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - } - if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){ - SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); - PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); - SDValue Load = - DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - } - } else { - // If we have any FPRs remaining, we may also have GPRs remaining. - // Args passed in FPRs consume either 1 (f32) or 2 (f64) available - // GPRs. - if (GPR_idx != NumGPRs) - ++GPR_idx; - if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && - !isPPC64) // PPC64 has 64-bit GPR's obviously :) - ++GPR_idx; - } - } else - LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, - isPPC64, CFlags.IsTailCall, false, MemOpChains, - TailCallArguments, dl); - if (isPPC64) - ArgOffset += 8; - else - ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8; - break; - case MVT::v4f32: - case MVT::v4i32: - case MVT::v8i16: - case MVT::v16i8: - if (CFlags.IsVarArg) { - // These go aligned on the stack, or in the corresponding R registers - // when within range. The Darwin PPC ABI doc claims they also go in - // V registers; in fact gcc does this only for arguments that are - // prototyped, not for those that match the ... We do it for all - // arguments, seems to work. - while (ArgOffset % 16 !=0) { - ArgOffset += PtrByteSize; - if (GPR_idx != NumGPRs) - GPR_idx++; - } - // We could elide this store in the case where the object fits - // entirely in R registers. Maybe later. - PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, - DAG.getConstant(ArgOffset, dl, PtrVT)); - SDValue Store = - DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Store); - if (VR_idx != NumVRs) { - SDValue Load = - DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); - } - ArgOffset += 16; - for (unsigned i=0; i<16; i+=PtrByteSize) { - if (GPR_idx == NumGPRs) - break; - SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, - DAG.getConstant(i, dl, PtrVT)); - SDValue Load = - DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); - MemOpChains.push_back(Load.getValue(1)); - RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); - } - break; - } - - // Non-varargs Altivec params generally go in registers, but have - // stack space allocated at the end. - if (VR_idx != NumVRs) { - // Doesn't have GPR space allocated. - RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); - } else if (nAltivecParamsAtEnd==0) { - // We are emitting Altivec params in order. - LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, - isPPC64, CFlags.IsTailCall, true, MemOpChains, - TailCallArguments, dl); - ArgOffset += 16; - } - break; - } - } - // If all Altivec parameters fit in registers, as they usually do, - // they get stack space following the non-Altivec parameters. We - // don't track this here because nobody below needs it. - // If there are more Altivec parameters than fit in registers emit - // the stores here. - if (!CFlags.IsVarArg && nAltivecParamsAtEnd > NumVRs) { - unsigned j = 0; - // Offset is aligned; skip 1st 12 params which go in V registers. - ArgOffset = ((ArgOffset+15)/16)*16; - ArgOffset += 12*16; - for (unsigned i = 0; i != NumOps; ++i) { - SDValue Arg = OutVals[i]; - EVT ArgType = Outs[i].VT; - if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || - ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { - if (++j > NumVRs) { - SDValue PtrOff; - // We are emitting Altivec params in order. - LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, - isPPC64, CFlags.IsTailCall, true, MemOpChains, - TailCallArguments, dl); - ArgOffset += 16; - } - } - } - } - - if (!MemOpChains.empty()) - Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); - - // On Darwin, R12 must contain the address of an indirect callee. This does - // not mean the MTCTR instruction must use R12; it's easier to model this as - // an extra parameter, so do that. - if (CFlags.IsIndirect) { - assert(!CFlags.IsTailCall && "Indirect tail-calls not supported."); - RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : - PPC::R12), Callee)); - } - - // Build a sequence of copy-to-reg nodes chained together with token chain - // and flag operands which copy the outgoing args into the appropriate regs. - SDValue InFlag; - for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { - Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, - RegsToPass[i].second, InFlag); - InFlag = Chain.getValue(1); - } - - if (CFlags.IsTailCall) - PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, - TailCallArguments); - - return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, - Callee, SPDiff, NumBytes, Ins, InVals, CB); -} - static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State) { @@ -7053,9 +6267,10 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, const Align PtrAlign = IsPPC64 ? Align(8) : Align(4); const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; - assert((!ValVT.isInteger() || - (ValVT.getSizeInBits() <= RegVT.getSizeInBits())) && - "Integer argument exceeds register size: should have been legalized"); + if (ValVT.isVector() && !State.getMachineFunction() + .getTarget() + .Options.EnableAIXExtendedAltivecABI) + report_fatal_error("the default Altivec AIX ABI is not yet supported"); if (ValVT == MVT::f128) report_fatal_error("f128 is unimplemented on AIX."); @@ -7063,9 +6278,6 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, if (ArgFlags.isNest()) report_fatal_error("Nest arguments are unimplemented."); - if (ValVT.isVector() || LocVT.isVector()) - report_fatal_error("Vector arguments are unimplemented on AIX."); - static const MCPhysReg GPR_32[] = {// 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10}; @@ -7073,6 +6285,11 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10}; + static const MCPhysReg VR[] = {// Vector registers. + PPC::V2, PPC::V3, PPC::V4, PPC::V5, + PPC::V6, PPC::V7, PPC::V8, PPC::V9, + PPC::V10, PPC::V11, PPC::V12, PPC::V13}; + if (ArgFlags.isByVal()) { if (ArgFlags.getNonZeroByValAlign() > PtrAlign) report_fatal_error("Pass-by-value arguments with alignment greater than " @@ -7117,7 +6334,7 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, case MVT::i32: { const unsigned Offset = State.AllocateStack(PtrAlign.value(), PtrAlign); // AIX integer arguments are always passed in register width. - if (ValVT.getSizeInBits() < RegVT.getSizeInBits()) + if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits()) LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt : CCValAssign::LocInfo::ZExt; if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) @@ -7168,6 +6385,25 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, return false; } + case MVT::v4f32: + case MVT::v4i32: + case MVT::v8i16: + case MVT::v16i8: + case MVT::v2i64: + case MVT::v2f64: + case MVT::v1i128: { + if (State.isVarArg()) + report_fatal_error( + "variadic arguments for vector types are unimplemented for AIX"); + + if (unsigned VReg = State.AllocateReg(VR)) + State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); + else { + report_fatal_error( + "passing vector parameters to the stack is unimplemented for AIX"); + } + return false; + } } return true; } @@ -7188,6 +6424,14 @@ static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, return &PPC::F4RCRegClass; case MVT::f64: return &PPC::F8RCRegClass; + case MVT::v4f32: + case MVT::v4i32: + case MVT::v8i16: + case MVT::v16i8: + case MVT::v2i64: + case MVT::v2f64: + case MVT::v1i128: + return &PPC::VRRCRegClass; } } @@ -7195,7 +6439,7 @@ static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT, SelectionDAG &DAG, SDValue ArgValue, MVT LocVT, const SDLoc &dl) { assert(ValVT.isScalarInteger() && LocVT.isScalarInteger()); - assert(ValVT.getSizeInBits() < LocVT.getSizeInBits()); + assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits()); if (Flags.isSExt()) ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue, @@ -7282,8 +6526,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(DAG.getSubtarget()); - if (Subtarget.hasQPX()) - report_fatal_error("QPX support is not supported on AIX."); const bool IsPPC64 = Subtarget.isPPC64(); const unsigned PtrByteSize = IsPPC64 ? 8 : 4; @@ -7292,6 +6534,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( SmallVector<CCValAssign, 16> ArgLocs; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); + PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); const EVT PtrVT = getPointerTy(MF.getDataLayout()); @@ -7306,6 +6549,9 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( CCValAssign &VA = ArgLocs[I++]; MVT LocVT = VA.getLocVT(); ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags; + if (VA.isMemLoc() && VA.getValVT().isVector()) + report_fatal_error( + "passing vector parameters to the stack is unimplemented for AIX"); // For compatibility with the AIX XL compiler, the float args in the // parameter save area are initialized even if the argument is available @@ -7316,6 +6562,15 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( if (VA.isMemLoc() && VA.needsCustom()) continue; + if (VA.isRegLoc()) { + if (VA.getValVT().isScalarInteger()) + FuncInfo->appendParameterType(PPCFunctionInfo::FixedType); + else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) + FuncInfo->appendParameterType(VA.getValVT().SimpleTy == MVT::f32 + ? PPCFunctionInfo::ShortFloatPoint + : PPCFunctionInfo::LongFloatPoint); + } + if (Flags.isByVal() && VA.isMemLoc()) { const unsigned Size = alignTo(Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize, @@ -7361,10 +6616,10 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( // to extracting the value from the register directly, and elide the // stores when the arguments address is not taken, but that will need to // be future work. - SDValue Store = - DAG.getStore(CopyFrom.getValue(1), dl, CopyFrom, - DAG.getObjectPtrOffset(dl, FIN, Offset), - MachinePointerInfo::getFixedStack(MF, FI, Offset)); + SDValue Store = DAG.getStore( + CopyFrom.getValue(1), dl, CopyFrom, + DAG.getObjectPtrOffset(dl, FIN, TypeSize::Fixed(Offset)), + MachinePointerInfo::getFixedStack(MF, FI, Offset)); MemOps.push_back(Store); }; @@ -7379,6 +6634,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( const CCValAssign RL = ArgLocs[I++]; HandleRegLoc(RL.getLocReg(), Offset); + FuncInfo->appendParameterType(PPCFunctionInfo::FixedType); } if (Offset != StackSize) { @@ -7400,7 +6656,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( MF.addLiveIn(VA.getLocReg(), getRegClassForSVT(SVT, IsPPC64)); SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); if (ValVT.isScalarInteger() && - (ValVT.getSizeInBits() < LocVT.getSizeInBits())) { + (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) { ArgValue = truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl); } @@ -7441,7 +6697,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( // aligned stack. CallerReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea); - PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); FuncInfo->setMinReservedArea(CallerReservedArea); if (isVarArg) { @@ -7503,10 +6758,6 @@ SDValue PPCTargetLowering::LowerCall_AIX( const PPCSubtarget& Subtarget = static_cast<const PPCSubtarget&>(DAG.getSubtarget()); - if (Subtarget.hasQPX()) - report_fatal_error("QPX is not supported on AIX."); - if (Subtarget.hasAltivec()) - report_fatal_error("Altivec support is unimplemented on AIX."); MachineFunction &MF = DAG.getMachineFunction(); SmallVector<CCValAssign, 16> ArgLocs; @@ -7563,11 +6814,12 @@ SDValue PPCTargetLowering::LowerCall_AIX( } auto GetLoad = [&](EVT VT, unsigned LoadOffset) { - return DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, - (LoadOffset != 0) - ? DAG.getObjectPtrOffset(dl, Arg, LoadOffset) - : Arg, - MachinePointerInfo(), VT); + return DAG.getExtLoad( + ISD::ZEXTLOAD, dl, PtrVT, Chain, + (LoadOffset != 0) + ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset)) + : Arg, + MachinePointerInfo(), VT); }; unsigned LoadOffset = 0; @@ -7597,9 +6849,11 @@ SDValue PPCTargetLowering::LowerCall_AIX( // Only memcpy the bytes that don't pass in register. MemcpyFlags.setByValSize(ByValSize - LoadOffset); Chain = CallSeqStart = createMemcpyOutsideCallSeq( - (LoadOffset != 0) ? DAG.getObjectPtrOffset(dl, Arg, LoadOffset) - : Arg, - DAG.getObjectPtrOffset(dl, StackPtr, ByValVA.getLocMemOffset()), + (LoadOffset != 0) + ? DAG.getObjectPtrOffset(dl, Arg, TypeSize::Fixed(LoadOffset)) + : Arg, + DAG.getObjectPtrOffset(dl, StackPtr, + TypeSize::Fixed(ByValVA.getLocMemOffset())), CallSeqStart, MemcpyFlags, DAG, dl); continue; } @@ -7649,6 +6903,10 @@ SDValue PPCTargetLowering::LowerCall_AIX( const MVT LocVT = VA.getLocVT(); const MVT ValVT = VA.getValVT(); + if (VA.isMemLoc() && VA.getValVT().isVector()) + report_fatal_error( + "passing vector parameters to the stack is unimplemented for AIX"); + switch (VA.getLocInfo()) { default: report_fatal_error("Unexpected argument extension type."); @@ -7690,7 +6948,8 @@ SDValue PPCTargetLowering::LowerCall_AIX( // f32 in 32-bit GPR // f64 in 64-bit GPR RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt)); - else if (Arg.getValueType().getSizeInBits() < LocVT.getSizeInBits()) + else if (Arg.getValueType().getFixedSizeInBits() < + LocVT.getFixedSizeInBits()) // f32 in 64-bit GPR. RegsToPass.push_back(std::make_pair( VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, LocVT))); @@ -8049,20 +7308,45 @@ SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1> - assert(Op.getValueType().isVector() && "Vector type expected."); - - SDLoc DL(Op); - SDValue N1 = Op.getOperand(0); - unsigned SrcSize = N1.getValueType().getSizeInBits(); - assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector"); - SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); - EVT TrgVT = Op.getValueType(); + assert(TrgVT.isVector() && "Vector type expected."); unsigned TrgNumElts = TrgVT.getVectorNumElements(); EVT EltVT = TrgVT.getVectorElementType(); + if (!isOperationCustom(Op.getOpcode(), TrgVT) || + TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(TrgNumElts) || + !isPowerOf2_32(EltVT.getSizeInBits())) + return SDValue(); + + SDValue N1 = Op.getOperand(0); + EVT SrcVT = N1.getValueType(); + unsigned SrcSize = SrcVT.getSizeInBits(); + if (SrcSize > 256 || + !isPowerOf2_32(SrcVT.getVectorNumElements()) || + !isPowerOf2_32(SrcVT.getVectorElementType().getSizeInBits())) + return SDValue(); + if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2) + return SDValue(); + unsigned WideNumElts = 128 / EltVT.getSizeInBits(); EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); + SDLoc DL(Op); + SDValue Op1, Op2; + if (SrcSize == 256) { + EVT VecIdxTy = getVectorIdxTy(DAG.getDataLayout()); + EVT SplitVT = + N1.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); + unsigned SplitNumElts = SplitVT.getVectorNumElements(); + Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1, + DAG.getConstant(0, DL, VecIdxTy)); + Op2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1, + DAG.getConstant(SplitNumElts, DL, VecIdxTy)); + } + else { + Op1 = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); + Op2 = DAG.getUNDEF(WideVT); + } + // First list the elements we want to keep. unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); SmallVector<int, 16> ShuffV; @@ -8078,16 +7362,17 @@ SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, // ShuffV.push_back(i + WideNumElts); ShuffV.push_back(WideNumElts + 1); - SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc); - return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV); + Op1 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op1); + Op2 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op2); + return DAG.getVectorShuffle(WideVT, DL, Op1, Op2, ShuffV); } /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when /// possible. SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { - // Not FP? Not a fsel. + // Not FP, or using SPE? Not a fsel. if (!Op.getOperand(0).getValueType().isFloatingPoint() || - !Op.getOperand(2).getValueType().isFloatingPoint()) + !Op.getOperand(2).getValueType().isFloatingPoint() || Subtarget.hasSPE()) return Op; ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); @@ -8203,54 +7488,105 @@ SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { return Op; } -void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, - SelectionDAG &DAG, - const SDLoc &dl) const { - assert(Op.getOperand(0).getValueType().isFloatingPoint()); - SDValue Src = Op.getOperand(0); - if (Src.getValueType() == MVT::f32) - Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); +static unsigned getPPCStrictOpcode(unsigned Opc) { + switch (Opc) { + default: + llvm_unreachable("No strict version of this opcode!"); + case PPCISD::FCTIDZ: + return PPCISD::STRICT_FCTIDZ; + case PPCISD::FCTIWZ: + return PPCISD::STRICT_FCTIWZ; + case PPCISD::FCTIDUZ: + return PPCISD::STRICT_FCTIDUZ; + case PPCISD::FCTIWUZ: + return PPCISD::STRICT_FCTIWUZ; + case PPCISD::FCFID: + return PPCISD::STRICT_FCFID; + case PPCISD::FCFIDU: + return PPCISD::STRICT_FCFIDU; + case PPCISD::FCFIDS: + return PPCISD::STRICT_FCFIDS; + case PPCISD::FCFIDUS: + return PPCISD::STRICT_FCFIDUS; + } +} + +static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG, + const PPCSubtarget &Subtarget) { + SDLoc dl(Op); + bool IsStrict = Op->isStrictFPOpcode(); + bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || + Op.getOpcode() == ISD::STRICT_FP_TO_SINT; - SDValue Tmp; + // TODO: Any other flags to propagate? + SDNodeFlags Flags; + Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); + + // For strict nodes, source is the second operand. + SDValue Src = Op.getOperand(IsStrict ? 1 : 0); + SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); + assert(Src.getValueType().isFloatingPoint()); + if (Src.getValueType() == MVT::f32) { + if (IsStrict) { + Src = + DAG.getNode(ISD::STRICT_FP_EXTEND, dl, + DAG.getVTList(MVT::f64, MVT::Other), {Chain, Src}, Flags); + Chain = Src.getValue(1); + } else + Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); + } + SDValue Conv; + unsigned Opc = ISD::DELETED_NODE; switch (Op.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); case MVT::i32: - Tmp = DAG.getNode( - Op.getOpcode() == ISD::FP_TO_SINT - ? PPCISD::FCTIWZ - : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), - dl, MVT::f64, Src); + Opc = IsSigned ? PPCISD::FCTIWZ + : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ); break; case MVT::i64: - assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && + assert((IsSigned || Subtarget.hasFPCVT()) && "i64 FP_TO_UINT is supported only with FPCVT"); - Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : - PPCISD::FCTIDUZ, - dl, MVT::f64, Src); - break; + Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ; + } + if (IsStrict) { + Opc = getPPCStrictOpcode(Opc); + Conv = DAG.getNode(Opc, dl, DAG.getVTList(MVT::f64, MVT::Other), + {Chain, Src}, Flags); + } else { + Conv = DAG.getNode(Opc, dl, MVT::f64, Src); } + return Conv; +} + +void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, + SelectionDAG &DAG, + const SDLoc &dl) const { + SDValue Tmp = convertFPToInt(Op, DAG, Subtarget); + bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || + Op.getOpcode() == ISD::STRICT_FP_TO_SINT; + bool IsStrict = Op->isStrictFPOpcode(); // Convert the FP value to an int value through memory. bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && - (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); + (IsSigned || Subtarget.hasFPCVT()); SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); // Emit a store to the stack slot. - SDValue Chain; + SDValue Chain = IsStrict ? Tmp.getValue(1) : DAG.getEntryNode(); Align Alignment(DAG.getEVTAlign(Tmp.getValueType())); if (i32Stack) { MachineFunction &MF = DAG.getMachineFunction(); Alignment = Align(4); MachineMemOperand *MMO = MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Alignment); - SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; + SDValue Ops[] = { Chain, Tmp, FIPtr }; Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); } else - Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI, Alignment); + Chain = DAG.getStore(Chain, dl, Tmp, FIPtr, MPI, Alignment); // Result is a load from the stack slot. If loading 4 bytes, make sure to // add in a bias on big endian. @@ -8272,76 +7608,100 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { - assert(Op.getOperand(0).getValueType().isFloatingPoint()); - SDValue Src = Op.getOperand(0); - - if (Src.getValueType() == MVT::f32) - Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); - - SDValue Tmp; - switch (Op.getSimpleValueType().SimpleTy) { - default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); - case MVT::i32: - Tmp = DAG.getNode( - Op.getOpcode() == ISD::FP_TO_SINT - ? PPCISD::FCTIWZ - : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), - dl, MVT::f64, Src); - Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); - break; - case MVT::i64: - assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && - "i64 FP_TO_UINT is supported only with FPCVT"); - Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : - PPCISD::FCTIDUZ, - dl, MVT::f64, Src); - Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); - break; - } - return Tmp; + SDValue Conv = convertFPToInt(Op, DAG, Subtarget); + SDValue Mov = DAG.getNode(PPCISD::MFVSR, dl, Op.getValueType(), Conv); + if (Op->isStrictFPOpcode()) + return DAG.getMergeValues({Mov, Conv.getValue(1)}, dl); + else + return Mov; } SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { + bool IsStrict = Op->isStrictFPOpcode(); + bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT || + Op.getOpcode() == ISD::STRICT_FP_TO_SINT; + SDValue Src = Op.getOperand(IsStrict ? 1 : 0); + EVT SrcVT = Src.getValueType(); + EVT DstVT = Op.getValueType(); // FP to INT conversions are legal for f128. - if (Op->getOperand(0).getValueType() == MVT::f128) - return Op; + if (SrcVT == MVT::f128) + return Subtarget.hasP9Vector() ? Op : SDValue(); // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on // PPC (the libcall is not available). - if (Op.getOperand(0).getValueType() == MVT::ppcf128) { - if (Op.getValueType() == MVT::i32) { - if (Op.getOpcode() == ISD::FP_TO_SINT) { - SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, - MVT::f64, Op.getOperand(0), + if (SrcVT == MVT::ppcf128) { + if (DstVT == MVT::i32) { + // TODO: Conservatively pass only nofpexcept flag here. Need to check and + // set other fast-math flags to FP operations in both strict and + // non-strict cases. (FP_TO_SINT, FSUB) + SDNodeFlags Flags; + Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); + + if (IsSigned) { + SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src, DAG.getIntPtrConstant(0, dl)); - SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, - MVT::f64, Op.getOperand(0), + SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Src, DAG.getIntPtrConstant(1, dl)); - // Add the two halves of the long double in round-to-zero mode. - SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); - - // Now use a smaller FP_TO_SINT. - return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); - } - if (Op.getOpcode() == ISD::FP_TO_UINT) { + // Add the two halves of the long double in round-to-zero mode, and use + // a smaller FP_TO_SINT. + if (IsStrict) { + SDValue Res = DAG.getNode(PPCISD::STRICT_FADDRTZ, dl, + DAG.getVTList(MVT::f64, MVT::Other), + {Op.getOperand(0), Lo, Hi}, Flags); + return DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, + DAG.getVTList(MVT::i32, MVT::Other), + {Res.getValue(1), Res}, Flags); + } else { + SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); + return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); + } + } else { const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); - SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128); - // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X - // FIXME: generated code sucks. - // TODO: Are there fast-math-flags to propagate to this FSUB? - SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, - Op.getOperand(0), Tmp); - True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); - True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, - DAG.getConstant(0x80000000, dl, MVT::i32)); - SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, - Op.getOperand(0)); - return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False, - ISD::SETGE); + SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT); + SDValue SignMask = DAG.getConstant(0x80000000, dl, DstVT); + if (IsStrict) { + // Sel = Src < 0x80000000 + // FltOfs = select Sel, 0.0, 0x80000000 + // IntOfs = select Sel, 0, 0x80000000 + // Result = fp_to_sint(Src - FltOfs) ^ IntOfs + SDValue Chain = Op.getOperand(0); + EVT SetCCVT = + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT); + EVT DstSetCCVT = + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT); + SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, + Chain, true); + Chain = Sel.getValue(1); + + SDValue FltOfs = DAG.getSelect( + dl, SrcVT, Sel, DAG.getConstantFP(0.0, dl, SrcVT), Cst); + Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); + + SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, + DAG.getVTList(SrcVT, MVT::Other), + {Chain, Src, FltOfs}, Flags); + Chain = Val.getValue(1); + SDValue SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, + DAG.getVTList(DstVT, MVT::Other), + {Chain, Val}, Flags); + Chain = SInt.getValue(1); + SDValue IntOfs = DAG.getSelect( + dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT), SignMask); + SDValue Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); + return DAG.getMergeValues({Result, Chain}, dl); + } else { + // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X + // FIXME: generated code sucks. + SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Src, Cst); + True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); + True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, SignMask); + SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src); + return DAG.getSelectCC(dl, Src, Cst, True, False, ISD::SETGE); + } } } @@ -8370,6 +7730,10 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, SelectionDAG &DAG, ISD::LoadExtType ET) const { + // Conservatively skip reusing for constrained FP nodes. + if (Op->isStrictFPOpcode()) + return false; + SDLoc dl(Op); bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT && (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32); @@ -8389,6 +7753,13 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, if (LD->getMemoryVT() != MemVT) return false; + // If the result of the load is an illegal type, then we can't build a + // valid chain for reuse since the legalised loads and token factor node that + // ties the legalised loads together uses a different output chain then the + // illegal load. + if (!isTypeLegal(LD->getValueType(0))) + return false; + RLI.Ptr = LD->getBasePtr(); if (LD->isIndexed() && !LD->getOffset().isUndef()) { assert(LD->getAddressingMode() == ISD::PRE_INC && @@ -8453,13 +7824,41 @@ bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { continue; if (UI->getOpcode() != ISD::SINT_TO_FP && - UI->getOpcode() != ISD::UINT_TO_FP) + UI->getOpcode() != ISD::UINT_TO_FP && + UI->getOpcode() != ISD::STRICT_SINT_TO_FP && + UI->getOpcode() != ISD::STRICT_UINT_TO_FP) return true; } return false; } +static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG, + const PPCSubtarget &Subtarget, + SDValue Chain = SDValue()) { + bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP || + Op.getOpcode() == ISD::STRICT_SINT_TO_FP; + SDLoc dl(Op); + + // TODO: Any other flags to propagate? + SDNodeFlags Flags; + Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); + + // If we have FCFIDS, then use it when converting to single-precision. + // Otherwise, convert to double-precision and then round. + bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT(); + unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS) + : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU); + EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64; + if (Op->isStrictFPOpcode()) { + if (!Chain) + Chain = Op.getOperand(0); + return DAG.getNode(getPPCStrictOpcode(ConvOpc), dl, + DAG.getVTList(ConvTy, MVT::Other), {Chain, Src}, Flags); + } else + return DAG.getNode(ConvOpc, dl, ConvTy, Src); +} + /// Custom lowers integer to floating point conversions to use /// the direct move instructions available in ISA 2.07 to avoid the /// need for load/store combinations. @@ -8471,25 +7870,13 @@ SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, "Invalid floating point type as target of conversion"); assert(Subtarget.hasFPCVT() && "Int to FP conversions with direct moves require FPCVT"); - SDValue FP; - SDValue Src = Op.getOperand(0); - bool SinglePrec = Op.getValueType() == MVT::f32; + SDValue Src = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0); bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; - bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; - unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : - (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); - - if (WordInt) { - FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, - dl, MVT::f64, Src); - FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); - } - else { - FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); - FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); - } - - return FP; + bool Signed = Op.getOpcode() == ISD::SINT_TO_FP || + Op.getOpcode() == ISD::STRICT_SINT_TO_FP; + unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA; + SDValue Mov = DAG.getNode(MovOpc, dl, MVT::f64, Src); + return convertIntToFP(Op, Mov, DAG, Subtarget); } static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { @@ -8514,17 +7901,23 @@ static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { - + bool IsStrict = Op->isStrictFPOpcode(); unsigned Opc = Op.getOpcode(); - assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) && + SDValue Src = Op.getOperand(IsStrict ? 1 : 0); + assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP || + Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) && "Unexpected conversion type"); assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && "Supports conversions to v2f64/v4f32 only."); - bool SignedConv = Opc == ISD::SINT_TO_FP; + // TODO: Any other flags to propagate? + SDNodeFlags Flags; + Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); + + bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP; bool FourEltRes = Op.getValueType() == MVT::v4f32; - SDValue Wide = widenVec(DAG, Op.getOperand(0), dl); + SDValue Wide = widenVec(DAG, Src, dl); EVT WideVT = Wide.getValueType(); unsigned WideNumElts = WideVT.getVectorNumElements(); MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; @@ -8549,7 +7942,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, SDValue Extend; if (SignedConv) { Arrange = DAG.getBitcast(IntermediateVT, Arrange); - EVT ExtVT = Op.getOperand(0).getValueType(); + EVT ExtVT = Src.getValueType(); if (Subtarget.hasP9Altivec()) ExtVT = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(), IntermediateVT.getVectorNumElements()); @@ -8559,14 +7952,27 @@ SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, } else Extend = DAG.getNode(ISD::BITCAST, dl, IntermediateVT, Arrange); + if (IsStrict) + return DAG.getNode(Opc, dl, DAG.getVTList(Op.getValueType(), MVT::Other), + {Op.getOperand(0), Extend}, Flags); + return DAG.getNode(Opc, dl, Op.getValueType(), Extend); } SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); + bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP || + Op.getOpcode() == ISD::STRICT_SINT_TO_FP; + bool IsStrict = Op->isStrictFPOpcode(); + SDValue Src = Op.getOperand(IsStrict ? 1 : 0); + SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); - EVT InVT = Op.getOperand(0).getValueType(); + // TODO: Any other flags to propagate? + SDNodeFlags Flags; + Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept()); + + EVT InVT = Src.getValueType(); EVT OutVT = Op.getValueType(); if (OutVT.isVector() && OutVT.isFloatingPoint() && isOperationCustom(Op.getOpcode(), InVT)) @@ -8574,37 +7980,21 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, // Conversions to f128 are legal. if (Op.getValueType() == MVT::f128) - return Op; - - if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { - if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) - return SDValue(); - - SDValue Value = Op.getOperand(0); - // The values are now known to be -1 (false) or 1 (true). To convert this - // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). - // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 - Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); - - SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); - - Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); - - if (Op.getValueType() != MVT::v4f64) - Value = DAG.getNode(ISD::FP_ROUND, dl, - Op.getValueType(), Value, - DAG.getIntPtrConstant(1, dl)); - return Value; - } + return Subtarget.hasP9Vector() ? Op : SDValue(); // Don't handle ppc_fp128 here; let it be lowered to a libcall. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); - if (Op.getOperand(0).getValueType() == MVT::i1) - return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), - DAG.getConstantFP(1.0, dl, Op.getValueType()), - DAG.getConstantFP(0.0, dl, Op.getValueType())); + if (Src.getValueType() == MVT::i1) { + SDValue Sel = DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src, + DAG.getConstantFP(1.0, dl, Op.getValueType()), + DAG.getConstantFP(0.0, dl, Op.getValueType())); + if (IsStrict) + return DAG.getMergeValues({Sel, Chain}, dl); + else + return Sel; + } // If we have direct moves, we can do all the conversion, skip the store/load // however, without FPCVT we can't do most conversions. @@ -8612,22 +8002,11 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, Subtarget.isPPC64() && Subtarget.hasFPCVT()) return LowerINT_TO_FPDirectMove(Op, DAG, dl); - assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && + assert((IsSigned || Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"); - // If we have FCFIDS, then use it when converting to single-precision. - // Otherwise, convert to double-precision and then round. - unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) - ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS - : PPCISD::FCFIDS) - : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU - : PPCISD::FCFID); - MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) - ? MVT::f32 - : MVT::f64; - - if (Op.getOperand(0).getValueType() == MVT::i64) { - SDValue SINT = Op.getOperand(0); + if (Src.getValueType() == MVT::i64) { + SDValue SINT = Src; // When converting to single-precision, we actually need to convert // to double-precision first and then round to single-precision. // To avoid double-rounding effects during that operation, we have @@ -8715,16 +8094,16 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, int FrameIdx = MFI.CreateStackObject(4, Align(4), false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - SDValue Store = - DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, - MachinePointerInfo::getFixedStack( - DAG.getMachineFunction(), FrameIdx)); + SDValue Store = DAG.getStore(Chain, dl, SINT.getOperand(0), FIdx, + MachinePointerInfo::getFixedStack( + DAG.getMachineFunction(), FrameIdx)); + Chain = Store; assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; - RLI.Chain = Store; + RLI.Chain = Chain; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = Align(4); @@ -8737,18 +8116,27 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, PPCISD::LFIWZX : PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); + Chain = Bits.getValue(1); } else Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); - SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); + SDValue FP = convertIntToFP(Op, Bits, DAG, Subtarget, Chain); + if (IsStrict) + Chain = FP.getValue(1); - if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) - FP = DAG.getNode(ISD::FP_ROUND, dl, - MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); + if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { + if (IsStrict) + FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl, + DAG.getVTList(MVT::f32, MVT::Other), + {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags); + else + FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, + DAG.getIntPtrConstant(0, dl)); + } return FP; } - assert(Op.getOperand(0).getValueType() == MVT::i32 && + assert(Src.getValueType() == MVT::i32 && "Unhandled INT_TO_FP type in custom expander!"); // Since we only generate this in 64-bit mode, we can take advantage of // 64-bit registers. In particular, sign extend the input value into the @@ -8762,21 +8150,20 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { ReuseLoadInfo RLI; bool ReusingLoad; - if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, - DAG))) { + if (!(ReusingLoad = canReuseLoadAddress(Src, MVT::i32, RLI, DAG))) { int FrameIdx = MFI.CreateStackObject(4, Align(4), false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - SDValue Store = - DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, - MachinePointerInfo::getFixedStack( - DAG.getMachineFunction(), FrameIdx)); + SDValue Store = DAG.getStore(Chain, dl, Src, FIdx, + MachinePointerInfo::getFixedStack( + DAG.getMachineFunction(), FrameIdx)); + Chain = Store; assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; - RLI.Chain = Store; + RLI.Chain = Chain; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = Align(4); @@ -8786,10 +8173,10 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; - Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? - PPCISD::LFIWZX : PPCISD::LFIWAX, - dl, DAG.getVTList(MVT::f64, MVT::Other), - Ops, MVT::i32, MMO); + Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl, + DAG.getVTList(MVT::f64, MVT::Other), Ops, + MVT::i32, MMO); + Chain = Ld.getValue(1); if (ReusingLoad) spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); } else { @@ -8799,25 +8186,34 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, int FrameIdx = MFI.CreateStackObject(8, Align(8), false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, - Op.getOperand(0)); + SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Src); // STD the extended value into the stack slot. SDValue Store = DAG.getStore( - DAG.getEntryNode(), dl, Ext64, FIdx, + Chain, dl, Ext64, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); + Chain = Store; // Load the value as a double. Ld = DAG.getLoad( - MVT::f64, dl, Store, FIdx, + MVT::f64, dl, Chain, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); + Chain = Ld.getValue(1); } // FCFID it and return it. - SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); - if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) - FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, - DAG.getIntPtrConstant(0, dl)); + SDValue FP = convertIntToFP(Op, Ld, DAG, Subtarget, Chain); + if (IsStrict) + Chain = FP.getValue(1); + if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { + if (IsStrict) + FP = DAG.getNode(ISD::STRICT_FP_ROUND, dl, + DAG.getVTList(MVT::f32, MVT::Other), + {Chain, FP, DAG.getIntPtrConstant(0, dl)}, Flags); + else + FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, + DAG.getIntPtrConstant(0, dl)); + } return FP; } @@ -8852,16 +8248,24 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, SDValue MFFS = DAG.getNode(PPCISD::MFFS, dl, {MVT::f64, MVT::Other}, Chain); Chain = MFFS.getValue(1); - // Save FP register to stack slot - int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false); - SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); - Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo()); + SDValue CWD; + if (isTypeLegal(MVT::i64)) { + CWD = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, + DAG.getNode(ISD::BITCAST, dl, MVT::i64, MFFS)); + } else { + // Save FP register to stack slot + int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false); + SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); + Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo()); - // Load FP Control Word from low 32 bits of stack slot. - SDValue Four = DAG.getConstant(4, dl, PtrVT); - SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); - SDValue CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo()); - Chain = CWD.getValue(1); + // Load FP Control Word from low 32 bits of stack slot. + assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) && + "Stack slot adjustment is valid only on big endian subtargets!"); + SDValue Four = DAG.getConstant(4, dl, PtrVT); + SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); + CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo()); + Chain = CWD.getValue(1); + } // Transform as necessary SDValue CWD1 = @@ -8972,6 +8376,31 @@ SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { return DAG.getMergeValues(OutOps, dl); } +SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op, + SelectionDAG &DAG) const { + SDLoc dl(Op); + EVT VT = Op.getValueType(); + unsigned BitWidth = VT.getSizeInBits(); + + bool IsFSHL = Op.getOpcode() == ISD::FSHL; + SDValue X = Op.getOperand(0); + SDValue Y = Op.getOperand(1); + SDValue Z = Op.getOperand(2); + EVT AmtVT = Z.getValueType(); + + // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) + // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) + // This is simpler than TargetLowering::expandFunnelShift because we can rely + // on PowerPC shift by BW being well defined. + Z = DAG.getNode(ISD::AND, dl, AmtVT, Z, + DAG.getConstant(BitWidth - 1, dl, AmtVT)); + SDValue SubZ = + DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Z); + X = DAG.getNode(PPCISD::SHL, dl, VT, X, IsFSHL ? Z : SubZ); + Y = DAG.getNode(PPCISD::SRL, dl, VT, Y, IsFSHL ? SubZ : Z); + return DAG.getNode(ISD::OR, dl, VT, X, Y); +} + //===----------------------------------------------------------------------===// // Vector related lowering. // @@ -8987,7 +8416,7 @@ static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT, EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize. - if (Val == ((1LU << (SplatSize * 8)) - 1)) { + if (Val == ((1LLU << (SplatSize * 8)) - 1)) { SplatSize = 1; Val = 0xFF; } @@ -9111,13 +8540,15 @@ SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { Op0.getOperand(1)); } -static const SDValue *getNormalLoadInput(const SDValue &Op) { +static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) { const SDValue *InputLoad = &Op; if (InputLoad->getOpcode() == ISD::BITCAST) InputLoad = &InputLoad->getOperand(0); if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR || - InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) + InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) { + IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED; InputLoad = &InputLoad->getOperand(0); + } if (InputLoad->getOpcode() != ISD::LOAD) return nullptr; LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); @@ -9163,110 +8594,6 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); - if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { - // We first build an i32 vector, load it into a QPX register, - // then convert it to a floating-point vector and compare it - // to a zero vector to get the boolean result. - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - int FrameIdx = MFI.CreateStackObject(16, Align(16), false); - MachinePointerInfo PtrInfo = - MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); - EVT PtrVT = getPointerTy(DAG.getDataLayout()); - SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - - assert(BVN->getNumOperands() == 4 && - "BUILD_VECTOR for v4i1 does not have 4 operands"); - - bool IsConst = true; - for (unsigned i = 0; i < 4; ++i) { - if (BVN->getOperand(i).isUndef()) continue; - if (!isa<ConstantSDNode>(BVN->getOperand(i))) { - IsConst = false; - break; - } - } - - if (IsConst) { - Constant *One = - ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); - Constant *NegOne = - ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); - - Constant *CV[4]; - for (unsigned i = 0; i < 4; ++i) { - if (BVN->getOperand(i).isUndef()) - CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); - else if (isNullConstant(BVN->getOperand(i))) - CV[i] = NegOne; - else - CV[i] = One; - } - - Constant *CP = ConstantVector::get(CV); - SDValue CPIdx = - DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), Align(16)); - - SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; - SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other}); - return DAG.getMemIntrinsicNode( - PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, - MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); - } - - SmallVector<SDValue, 4> Stores; - for (unsigned i = 0; i < 4; ++i) { - if (BVN->getOperand(i).isUndef()) continue; - - unsigned Offset = 4*i; - SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); - Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); - - unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); - if (StoreSize > 4) { - Stores.push_back( - DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx, - PtrInfo.getWithOffset(Offset), MVT::i32)); - } else { - SDValue StoreValue = BVN->getOperand(i); - if (StoreSize < 4) - StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); - - Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx, - PtrInfo.getWithOffset(Offset))); - } - } - - SDValue StoreChain; - if (!Stores.empty()) - StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); - else - StoreChain = DAG.getEntryNode(); - - // Now load from v4i32 into the QPX register; this will extend it to - // v4i64 but not yet convert it to a floating point. Nevertheless, this - // is typed as v4f64 because the QPX register integer states are not - // explicitly represented. - - SDValue Ops[] = {StoreChain, - DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32), - FIdx}; - SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other}); - - SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, - dl, VTs, Ops, MVT::v4i32, PtrInfo); - LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, - DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), - LoadedVect); - - SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64); - - return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); - } - - // All other QPX vectors are handled by generic code. - if (Subtarget.hasQPX()) - return SDValue(); - // Check if this is a splat of a constant value. APInt APSplatBits, APSplatUndef; unsigned SplatBitSize; @@ -9277,19 +8604,48 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // If it is a splat of a double, check if we can shrink it to a 32 bit // non-denormal float which when converted back to double gives us the same - // double. This is to exploit the XXSPLTIDP instruction. - if (BVNIsConstantSplat && Subtarget.hasPrefixInstrs() && - (SplatBitSize == 64) && (Op->getValueType(0) == MVT::v2f64) && - convertToNonDenormSingle(APSplatBits)) { - SDValue SplatNode = DAG.getNode( - PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64, - DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32)); - return DAG.getBitcast(Op.getValueType(), SplatNode); + // double. This is to exploit the XXSPLTIDP instruction.+ // If we lose precision, we use XXSPLTI32DX. + if (BVNIsConstantSplat && (SplatBitSize == 64) && + Subtarget.hasPrefixInstrs()) { + if (convertToNonDenormSingle(APSplatBits) && + (Op->getValueType(0) == MVT::v2f64)) { + SDValue SplatNode = DAG.getNode( + PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64, + DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32)); + return DAG.getBitcast(Op.getValueType(), SplatNode); + } else { // We may lose precision, so we have to use XXSPLTI32DX. + + uint32_t Hi = + (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32); + uint32_t Lo = + (uint32_t)(APSplatBits.getZExtValue() & 0xFFFFFFFF); + SDValue SplatNode = DAG.getUNDEF(MVT::v2i64); + + if (!Hi || !Lo) + // If either load is 0, then we should generate XXLXOR to set to 0. + SplatNode = DAG.getTargetConstant(0, dl, MVT::v2i64); + + if (Hi) + SplatNode = DAG.getNode( + PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode, + DAG.getTargetConstant(0, dl, MVT::i32), + DAG.getTargetConstant(Hi, dl, MVT::i32)); + + if (Lo) + SplatNode = + DAG.getNode(PPCISD::XXSPLTI32DX, dl, MVT::v2i64, SplatNode, + DAG.getTargetConstant(1, dl, MVT::i32), + DAG.getTargetConstant(Lo, dl, MVT::i32)); + + return DAG.getBitcast(Op.getValueType(), SplatNode); + } } if (!BVNIsConstantSplat || SplatBitSize > 32) { - const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0)); + bool IsPermutedLoad = false; + const SDValue *InputLoad = + getNormalLoadInput(Op.getOperand(0), IsPermutedLoad); // Handle load-and-splat patterns as we have instructions that will do this // in one go. if (InputLoad && DAG.isSplatValue(Op, true)) { @@ -9301,7 +8657,12 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // Checking for a single use of this load, we have to check for vector // width (128 bits) / ElementSize uses (since each operand of the // BUILD_VECTOR is a separate use of the value. - if (InputLoad->getNode()->hasNUsesOfValue(128 / ElementSize, 0) && + unsigned NumUsesOfInputLD = 128 / ElementSize; + for (SDValue BVInOp : Op->ops()) + if (BVInOp.isUndef()) + NumUsesOfInputLD--; + assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?"); + if (InputLoad->getNode()->hasNUsesOfValue(NumUsesOfInputLD, 0) && ((Subtarget.hasVSX() && ElementSize == 64) || (Subtarget.hasP9Vector() && ElementSize == 32))) { SDValue Ops[] = { @@ -9309,17 +8670,21 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, LD->getBasePtr(), // Ptr DAG.getValueType(Op.getValueType()) // VT }; - return - DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, - DAG.getVTList(Op.getValueType(), MVT::Other), - Ops, LD->getMemoryVT(), LD->getMemOperand()); + SDValue LdSplt = DAG.getMemIntrinsicNode( + PPCISD::LD_SPLAT, dl, DAG.getVTList(Op.getValueType(), MVT::Other), + Ops, LD->getMemoryVT(), LD->getMemOperand()); + // Replace all uses of the output chain of the original load with the + // output chain of the new load. + DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1), + LdSplt.getValue(1)); + return LdSplt; } } - // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be - // lowered to VSX instructions under certain conditions. + // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to + // 32-bits can be lowered to VSX instructions under certain conditions. // Without VSX, there is no pattern more efficient than expanding the node. - if (Subtarget.hasVSX() && + if (Subtarget.hasVSX() && Subtarget.isPPC64() && haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), Subtarget.hasP8Vector())) return Op; @@ -9348,7 +8713,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be // turned into a 4-byte splat of 0xABABABAB. if (Subtarget.hasPrefixInstrs() && SplatSize == 2) - return getCanonicalConstSplat((SplatBits |= SplatBits << 16), SplatSize * 2, + return getCanonicalConstSplat(SplatBits | (SplatBits << 16), SplatSize * 2, Op.getValueType(), DAG, dl); if (Subtarget.hasPrefixInstrs() && SplatSize == 4) @@ -9444,17 +8809,6 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } - // vsplti + sra self. - if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { - SDValue Res = getCanonicalConstSplat(i, SplatSize, MVT::Other, DAG, dl); - static const unsigned IIDs[] = { // Intrinsic to use for each size. - Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, - Intrinsic::ppc_altivec_vsraw - }; - Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); - return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); - } - // vsplti + rol self. if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { @@ -9912,7 +9266,8 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, // If this is a load-and-splat, we can do that with a single instruction // in some cases. However if the load has multiple uses, we don't want to // combine it because that will just produce multiple loads. - const SDValue *InputLoad = getNormalLoadInput(V1); + bool IsPermutedLoad = false; + const SDValue *InputLoad = getNormalLoadInput(V1, IsPermutedLoad); if (InputLoad && Subtarget.hasVSX() && V2.isUndef() && (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) && InputLoad->hasOneUse()) { @@ -9920,6 +9275,16 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG); + // The splat index for permuted loads will be in the left half of the vector + // which is strictly wider than the loaded value by 8 bytes. So we need to + // adjust the splat index to point to the correct address in memory. + if (IsPermutedLoad) { + assert(isLittleEndian && "Unexpected permuted load on big endian target"); + SplatIdx += IsFourByte ? 2 : 1; + assert((SplatIdx < (IsFourByte ? 4 : 2)) && + "Splat of a value outside of the loaded memory"); + } + LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); // For 4-byte load-and-splat, we need Power9. if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) { @@ -9929,10 +9294,6 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, else Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8; - // If we are loading a partial vector, it does not make sense to adjust - // the base pointer. This happens with (splat (s_to_v_permuted (ld))). - if (LD->getMemoryVT().getSizeInBits() == (IsFourByte ? 32 : 64)) - Offset = 0; SDValue BasePtr = LD->getBasePtr(); if (Offset != 0) BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), @@ -9947,6 +9308,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SDValue LdSplt = DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL, Ops, LD->getMemoryVT(), LD->getMemOperand()); + DAG.ReplaceAllUsesOfValueWith(InputLoad->getValue(1), LdSplt.getValue(1)); if (LdSplt.getValueType() != SVOp->getValueType(0)) LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt); return LdSplt; @@ -10050,42 +9412,6 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, } } - if (Subtarget.hasQPX()) { - if (VT.getVectorNumElements() != 4) - return SDValue(); - - if (V2.isUndef()) V2 = V1; - - int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); - if (AlignIdx != -1) { - return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, - DAG.getConstant(AlignIdx, dl, MVT::i32)); - } else if (SVOp->isSplat()) { - int SplatIdx = SVOp->getSplatIndex(); - if (SplatIdx >= 4) { - std::swap(V1, V2); - SplatIdx -= 4; - } - - return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, - DAG.getConstant(SplatIdx, dl, MVT::i32)); - } - - // Lower this into a qvgpci/qvfperm pair. - - // Compute the qvgpci literal - unsigned idx = 0; - for (unsigned i = 0; i < 4; ++i) { - int m = SVOp->getMaskElt(i); - unsigned mm = m >= 0 ? (unsigned) m : i; - idx |= mm << (3-i)*3; - } - - SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, - DAG.getConstant(idx, dl, MVT::i32)); - return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); - } - // Cases that are handled by instructions that take permute immediates // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be // selected by the instruction selector. @@ -10347,6 +9673,26 @@ static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, return false; break; + case Intrinsic::ppc_altivec_vcmpequq: + case Intrinsic::ppc_altivec_vcmpgtsq: + case Intrinsic::ppc_altivec_vcmpgtuq: + if (!Subtarget.isISA3_1()) + return false; + switch (IntrinsicID) { + default: + llvm_unreachable("Unknown comparison intrinsic."); + case Intrinsic::ppc_altivec_vcmpequq: + CompareOpc = 455; + break; + case Intrinsic::ppc_altivec_vcmpgtsq: + CompareOpc = 903; + break; + case Intrinsic::ppc_altivec_vcmpgtuq: + CompareOpc = 647; + break; + } + break; + // VSX predicate comparisons use the same infrastructure case Intrinsic::ppc_vsx_xvcmpeqdp_p: case Intrinsic::ppc_vsx_xvcmpgedp_p: @@ -10470,6 +9816,26 @@ static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, else return false; break; + case Intrinsic::ppc_altivec_vcmpequq_p: + case Intrinsic::ppc_altivec_vcmpgtsq_p: + case Intrinsic::ppc_altivec_vcmpgtuq_p: + if (!Subtarget.isISA3_1()) + return false; + switch (IntrinsicID) { + default: + llvm_unreachable("Unknown comparison intrinsic."); + case Intrinsic::ppc_altivec_vcmpequq_p: + CompareOpc = 455; + break; + case Intrinsic::ppc_altivec_vcmpgtsq_p: + CompareOpc = 903; + break; + case Intrinsic::ppc_altivec_vcmpgtuq_p: + CompareOpc = 647; + break; + } + isDot = true; + break; } return true; } @@ -10483,11 +9849,32 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDLoc dl(Op); - if (IntrinsicID == Intrinsic::thread_pointer) { + switch (IntrinsicID) { + case Intrinsic::thread_pointer: // Reads the thread pointer register, used for __builtin_thread_pointer. if (Subtarget.isPPC64()) return DAG.getRegister(PPC::X13, MVT::i64); return DAG.getRegister(PPC::R2, MVT::i32); + + case Intrinsic::ppc_mma_disassemble_acc: + case Intrinsic::ppc_vsx_disassemble_pair: { + int NumVecs = 2; + SDValue WideVec = Op.getOperand(1); + if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) { + NumVecs = 4; + WideVec = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, WideVec); + } + SmallVector<SDValue, 4> RetOps; + for (int VecNo = 0; VecNo < NumVecs; VecNo++) { + SDValue Extract = DAG.getNode( + PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec, + DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo + : VecNo, + dl, MVT::i64)); + RetOps.push_back(Extract); + } + return DAG.getMergeValues(RetOps, dl); + } } // If this is a lowered altivec predicate compare, CompareOpc is set to the @@ -10512,7 +9899,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; - SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); + SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops); // Now that we have the comparison, emit a copy from the CR to a GPR. // This is flagged to the above dot comparison. @@ -10673,154 +10060,51 @@ SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, return Op; } -SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, - SelectionDAG &DAG) const { - SDLoc dl(Op); - SDNode *N = Op.getNode(); - - assert(N->getOperand(0).getValueType() == MVT::v4i1 && - "Unknown extract_vector_elt type"); - - SDValue Value = N->getOperand(0); - - // The first part of this is like the store lowering except that we don't - // need to track the chain. - - // The values are now known to be -1 (false) or 1 (true). To convert this - // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). - // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 - Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); - - // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to - // understand how to form the extending load. - SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); - - Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); - - // Now convert to an integer and store. - Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, - DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), - Value); - - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - int FrameIdx = MFI.CreateStackObject(16, Align(16), false); - MachinePointerInfo PtrInfo = - MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); - EVT PtrVT = getPointerTy(DAG.getDataLayout()); - SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - - SDValue StoreChain = DAG.getEntryNode(); - SDValue Ops[] = {StoreChain, - DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), - Value, FIdx}; - SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); - - StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, - dl, VTs, Ops, MVT::v4i32, PtrInfo); - - // Extract the value requested. - unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); - SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); - Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); - - SDValue IntVal = - DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); - - if (!Subtarget.useCRBits()) - return IntVal; - - return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); -} - -/// Lowering for QPX v4i1 loads SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); SDValue LoadChain = LN->getChain(); SDValue BasePtr = LN->getBasePtr(); + EVT VT = Op.getValueType(); - if (Op.getValueType() == MVT::v4f64 || - Op.getValueType() == MVT::v4f32) { - EVT MemVT = LN->getMemoryVT(); - unsigned Alignment = LN->getAlignment(); - - // If this load is properly aligned, then it is legal. - if (Alignment >= MemVT.getStoreSize()) - return Op; - - EVT ScalarVT = Op.getValueType().getScalarType(), - ScalarMemVT = MemVT.getScalarType(); - unsigned Stride = ScalarMemVT.getStoreSize(); - - SDValue Vals[4], LoadChains[4]; - for (unsigned Idx = 0; Idx < 4; ++Idx) { - SDValue Load; - if (ScalarVT != ScalarMemVT) - Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, - BasePtr, - LN->getPointerInfo().getWithOffset(Idx * Stride), - ScalarMemVT, MinAlign(Alignment, Idx * Stride), - LN->getMemOperand()->getFlags(), LN->getAAInfo()); - else - Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, - LN->getPointerInfo().getWithOffset(Idx * Stride), - MinAlign(Alignment, Idx * Stride), - LN->getMemOperand()->getFlags(), LN->getAAInfo()); - - if (Idx == 0 && LN->isIndexed()) { - assert(LN->getAddressingMode() == ISD::PRE_INC && - "Unknown addressing mode on vector load"); - Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), - LN->getAddressingMode()); - } - - Vals[Idx] = Load; - LoadChains[Idx] = Load.getValue(1); - - BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, - DAG.getConstant(Stride, dl, - BasePtr.getValueType())); - } - - SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); - SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals); - - if (LN->isIndexed()) { - SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; - return DAG.getMergeValues(RetOps, dl); - } + if (VT != MVT::v256i1 && VT != MVT::v512i1) + return Op; - SDValue RetOps[] = { Value, TF }; - return DAG.getMergeValues(RetOps, dl); + // Type v256i1 is used for pairs and v512i1 is used for accumulators. + // Here we create 2 or 4 v16i8 loads to load the pair or accumulator value in + // 2 or 4 vsx registers. + assert((VT != MVT::v512i1 || Subtarget.hasMMA()) && + "Type unsupported without MMA"); + assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && + "Type unsupported without paired vector support"); + Align Alignment = LN->getAlign(); + SmallVector<SDValue, 4> Loads; + SmallVector<SDValue, 4> LoadChains; + unsigned NumVecs = VT.getSizeInBits() / 128; + for (unsigned Idx = 0; Idx < NumVecs; ++Idx) { + SDValue Load = + DAG.getLoad(MVT::v16i8, dl, LoadChain, BasePtr, + LN->getPointerInfo().getWithOffset(Idx * 16), + commonAlignment(Alignment, Idx * 16), + LN->getMemOperand()->getFlags(), LN->getAAInfo()); + BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, + DAG.getConstant(16, dl, BasePtr.getValueType())); + Loads.push_back(Load); + LoadChains.push_back(Load.getValue(1)); } - - assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); - assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); - - // To lower v4i1 from a byte array, we load the byte elements of the - // vector and then reuse the BUILD_VECTOR logic. - - SDValue VectElmts[4], VectElmtChains[4]; - for (unsigned i = 0; i < 4; ++i) { - SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); - Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); - - VectElmts[i] = DAG.getExtLoad( - ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx, - LN->getPointerInfo().getWithOffset(i), MVT::i8, - /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo()); - VectElmtChains[i] = VectElmts[i].getValue(1); + if (Subtarget.isLittleEndian()) { + std::reverse(Loads.begin(), Loads.end()); + std::reverse(LoadChains.begin(), LoadChains.end()); } - - LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); - SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts); - - SDValue RVals[] = { Value, LoadChain }; - return DAG.getMergeValues(RVals, dl); + SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); + SDValue Value = + DAG.getNode(VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD, + dl, VT, Loads); + SDValue RetOps[] = {Value, TF}; + return DAG.getMergeValues(RetOps, dl); } -/// Lowering for QPX v4i1 stores SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); @@ -10828,122 +10112,40 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, SDValue StoreChain = SN->getChain(); SDValue BasePtr = SN->getBasePtr(); SDValue Value = SN->getValue(); + EVT StoreVT = Value.getValueType(); - if (Value.getValueType() == MVT::v4f64 || - Value.getValueType() == MVT::v4f32) { - EVT MemVT = SN->getMemoryVT(); - unsigned Alignment = SN->getAlignment(); - - // If this store is properly aligned, then it is legal. - if (Alignment >= MemVT.getStoreSize()) - return Op; - - EVT ScalarVT = Value.getValueType().getScalarType(), - ScalarMemVT = MemVT.getScalarType(); - unsigned Stride = ScalarMemVT.getStoreSize(); - - SDValue Stores[4]; - for (unsigned Idx = 0; Idx < 4; ++Idx) { - SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, - DAG.getVectorIdxConstant(Idx, dl)); - SDValue Store; - if (ScalarVT != ScalarMemVT) - Store = - DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, - SN->getPointerInfo().getWithOffset(Idx * Stride), - ScalarMemVT, MinAlign(Alignment, Idx * Stride), - SN->getMemOperand()->getFlags(), SN->getAAInfo()); - else - Store = DAG.getStore(StoreChain, dl, Ex, BasePtr, - SN->getPointerInfo().getWithOffset(Idx * Stride), - MinAlign(Alignment, Idx * Stride), - SN->getMemOperand()->getFlags(), SN->getAAInfo()); - - if (Idx == 0 && SN->isIndexed()) { - assert(SN->getAddressingMode() == ISD::PRE_INC && - "Unknown addressing mode on vector store"); - Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), - SN->getAddressingMode()); - } - - BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, - DAG.getConstant(Stride, dl, - BasePtr.getValueType())); - Stores[Idx] = Store; - } - - SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); - - if (SN->isIndexed()) { - SDValue RetOps[] = { TF, Stores[0].getValue(1) }; - return DAG.getMergeValues(RetOps, dl); - } + if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1) + return Op; - return TF; + // Type v256i1 is used for pairs and v512i1 is used for accumulators. + // Here we create 2 or 4 v16i8 stores to store the pair or accumulator + // underlying registers individually. + assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) && + "Type unsupported without MMA"); + assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) && + "Type unsupported without paired vector support"); + Align Alignment = SN->getAlign(); + SmallVector<SDValue, 4> Stores; + unsigned NumVecs = 2; + if (StoreVT == MVT::v512i1) { + Value = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, Value); + NumVecs = 4; } - - assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); - assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); - - // The values are now known to be -1 (false) or 1 (true). To convert this - // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). - // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 - Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); - - // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to - // understand how to form the extending load. - SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); - - Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); - - // Now convert to an integer and store. - Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, - DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), - Value); - - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - int FrameIdx = MFI.CreateStackObject(16, Align(16), false); - MachinePointerInfo PtrInfo = - MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); - EVT PtrVT = getPointerTy(DAG.getDataLayout()); - SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); - - SDValue Ops[] = {StoreChain, - DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), - Value, FIdx}; - SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); - - StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, - dl, VTs, Ops, MVT::v4i32, PtrInfo); - - // Move data into the byte array. - SDValue Loads[4], LoadChains[4]; - for (unsigned i = 0; i < 4; ++i) { - unsigned Offset = 4*i; - SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); - Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); - - Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, - PtrInfo.getWithOffset(Offset)); - LoadChains[i] = Loads[i].getValue(1); + for (unsigned Idx = 0; Idx < NumVecs; ++Idx) { + unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx; + SDValue Elt = DAG.getNode(PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, Value, + DAG.getConstant(VecNum, dl, MVT::i64)); + SDValue Store = + DAG.getStore(StoreChain, dl, Elt, BasePtr, + SN->getPointerInfo().getWithOffset(Idx * 16), + commonAlignment(Alignment, Idx * 16), + SN->getMemOperand()->getFlags(), SN->getAAInfo()); + BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, + DAG.getConstant(16, dl, BasePtr.getValueType())); + Stores.push_back(Store); } - - StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); - - SDValue Stores[4]; - for (unsigned i = 0; i < 4; ++i) { - SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); - Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); - - Stores[i] = DAG.getTruncStore( - StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), - MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(), - SN->getAAInfo()); - } - - StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); - - return StoreChain; + SDValue TF = DAG.getTokenFactor(dl, Stores); + return TF; } SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { @@ -11010,42 +10212,13 @@ SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { } } -SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const { - - assert(Op.getOpcode() == ISD::ABS && "Should only be called for ISD::ABS"); - - EVT VT = Op.getValueType(); - assert(VT.isVector() && - "Only set vector abs as custom, scalar abs shouldn't reach here!"); - assert((VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || - VT == MVT::v16i8) && - "Unexpected vector element type!"); - assert((VT != MVT::v2i64 || Subtarget.hasP8Altivec()) && - "Current subtarget doesn't support smax v2i64!"); - - // For vector abs, it can be lowered to: - // abs x - // ==> - // y = -x - // smax(x, y) - - SDLoc dl(Op); - SDValue X = Op.getOperand(0); - SDValue Zero = DAG.getConstant(0, dl, VT); - SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X); - - // SMAX patch https://reviews.llvm.org/D47332 - // hasn't landed yet, so use intrinsic first here. - // TODO: Should use SMAX directly once SMAX patch landed - Intrinsic::ID BifID = Intrinsic::ppc_altivec_vmaxsw; - if (VT == MVT::v2i64) - BifID = Intrinsic::ppc_altivec_vmaxsd; - else if (VT == MVT::v8i16) - BifID = Intrinsic::ppc_altivec_vmaxsh; - else if (VT == MVT::v16i8) - BifID = Intrinsic::ppc_altivec_vmaxsb; +SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { + bool IsStrict = Op->isStrictFPOpcode(); + if (Op.getOperand(IsStrict ? 1 : 0).getValueType() == MVT::f128 && + !Subtarget.hasP9Vector()) + return SDValue(); - return BuildIntrinsicOp(BifID, X, Y, DAG, dl, VT); + return Op; } // Custom lowering for fpext vf32 to v2f64 @@ -11158,8 +10331,12 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::STORE: return LowerSTORE(Op, DAG); case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); + case ISD::STRICT_FP_TO_UINT: + case ISD::STRICT_FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); + case ISD::STRICT_UINT_TO_FP: + case ISD::STRICT_SINT_TO_FP: case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); @@ -11169,16 +10346,20 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); + case ISD::FSHL: return LowerFunnelShift(Op, DAG); + case ISD::FSHR: return LowerFunnelShift(Op, DAG); + // Vector-related lowering. case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); - case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); case ISD::MUL: return LowerMUL(Op, DAG); - case ISD::ABS: return LowerABS(Op, DAG); case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); + case ISD::STRICT_FP_ROUND: + case ISD::FP_ROUND: + return LowerFP_ROUND(Op, DAG); case ISD::ROTL: return LowerROTL(Op, DAG); // For counter-based loop handling. @@ -11246,23 +10427,28 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N, } return; } + case ISD::STRICT_FP_TO_SINT: + case ISD::STRICT_FP_TO_UINT: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: // LowerFP_TO_INT() can only handle f32 and f64. - if (N->getOperand(0).getValueType() == MVT::ppcf128) + if (N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType() == + MVT::ppcf128) return; Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); return; case ISD::TRUNCATE: { - EVT TrgVT = N->getValueType(0); - EVT OpVT = N->getOperand(0).getValueType(); - if (TrgVT.isVector() && - isOperationCustom(N->getOpcode(), TrgVT) && - OpVT.getSizeInBits() <= 128 && - isPowerOf2_32(OpVT.getVectorElementType().getSizeInBits())) - Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG)); + if (!N->getValueType(0).isVector()) + return; + SDValue Lowered = LowerTRUNCATEVector(SDValue(N, 0), DAG); + if (Lowered) + Results.push_back(Lowered); return; } + case ISD::FSHL: + case ISD::FSHR: + // Don't handle funnel shifts here. + return; case ISD::BITCAST: // Don't handle bitcast here. return; @@ -11434,17 +10620,88 @@ PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, return BB; } +static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) { + switch(MI.getOpcode()) { + default: + return false; + case PPC::COPY: + return TII->isSignExtended(MI); + case PPC::LHA: + case PPC::LHA8: + case PPC::LHAU: + case PPC::LHAU8: + case PPC::LHAUX: + case PPC::LHAUX8: + case PPC::LHAX: + case PPC::LHAX8: + case PPC::LWA: + case PPC::LWAUX: + case PPC::LWAX: + case PPC::LWAX_32: + case PPC::LWA_32: + case PPC::PLHA: + case PPC::PLHA8: + case PPC::PLHA8pc: + case PPC::PLHApc: + case PPC::PLWA: + case PPC::PLWA8: + case PPC::PLWA8pc: + case PPC::PLWApc: + case PPC::EXTSB: + case PPC::EXTSB8: + case PPC::EXTSB8_32_64: + case PPC::EXTSB8_rec: + case PPC::EXTSB_rec: + case PPC::EXTSH: + case PPC::EXTSH8: + case PPC::EXTSH8_32_64: + case PPC::EXTSH8_rec: + case PPC::EXTSH_rec: + case PPC::EXTSW: + case PPC::EXTSWSLI: + case PPC::EXTSWSLI_32_64: + case PPC::EXTSWSLI_32_64_rec: + case PPC::EXTSWSLI_rec: + case PPC::EXTSW_32: + case PPC::EXTSW_32_64: + case PPC::EXTSW_32_64_rec: + case PPC::EXTSW_rec: + case PPC::SRAW: + case PPC::SRAWI: + case PPC::SRAWI_rec: + case PPC::SRAW_rec: + return true; + } + return false; +} + MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( MachineInstr &MI, MachineBasicBlock *BB, bool is8bit, // operation unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { + // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. + const PPCInstrInfo *TII = Subtarget.getInstrInfo(); + + // If this is a signed comparison and the value being compared is not known + // to be sign extended, sign extend it here. + DebugLoc dl = MI.getDebugLoc(); + MachineFunction *F = BB->getParent(); + MachineRegisterInfo &RegInfo = F->getRegInfo(); + Register incr = MI.getOperand(3).getReg(); + bool IsSignExtended = Register::isVirtualRegister(incr) && + isSignExtended(*RegInfo.getVRegDef(incr), TII); + + if (CmpOpcode == PPC::CMPW && !IsSignExtended) { + Register ValueReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); + BuildMI(*BB, MI, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueReg) + .addReg(MI.getOperand(3).getReg()); + MI.getOperand(3).setReg(ValueReg); + } // If we support part-word atomic mnemonics, just use them if (Subtarget.hasPartwordAtomics()) return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, CmpPred); - // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. - const TargetInstrInfo *TII = Subtarget.getInstrInfo(); // In 64 bit mode we have to use 64 bits for addresses, even though the // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address // registers without caring whether they're 32 or 64, but here we're @@ -11454,14 +10711,11 @@ MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; const BasicBlock *LLVM_BB = BB->getBasicBlock(); - MachineFunction *F = BB->getParent(); MachineFunction::iterator It = ++BB->getIterator(); Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); - Register incr = MI.getOperand(3).getReg(); - DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = @@ -11475,7 +10729,6 @@ MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); - MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; @@ -11950,18 +11203,34 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI, Register SPReg = isPPC64 ? PPC::X1 : PPC::R1; Register FinalStackPtr = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); Register FramePointer = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); + Register ActualNegSizeReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); - // Get the canonical FinalStackPtr like what - // PPCRegisterInfo::lowerDynamicAlloc does. - BuildMI(*MBB, {MI}, DL, - TII->get(isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 - : PPC::PREPARE_PROBED_ALLOCA_32), - FramePointer) - .addDef(FinalStackPtr) + // Since value of NegSizeReg might be realigned in prologepilog, insert a + // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and + // NegSize. + unsigned ProbeOpc; + if (!MRI.hasOneNonDBGUse(NegSizeReg)) + ProbeOpc = + isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32; + else + // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg + // and NegSizeReg will be allocated in the same phyreg to avoid + // redundant copy when NegSizeReg has only one use which is current MI and + // will be replaced by PREPARE_PROBED_ALLOCA then. + ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64 + : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32; + BuildMI(*MBB, {MI}, DL, TII->get(ProbeOpc), FramePointer) + .addDef(ActualNegSizeReg) .addReg(NegSizeReg) .add(MI.getOperand(2)) .add(MI.getOperand(3)); + // Calculate final stack pointer, which equals to SP + ActualNegSize. + BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4), + FinalStackPtr) + .addReg(SPReg) + .addReg(ActualNegSizeReg); + // Materialize a scratch register for update. int64_t NegProbeSize = -(int64_t)ProbeSize; assert(isInt<32>(NegProbeSize) && "Unhandled probe size!"); @@ -11982,7 +11251,7 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI, // Probing leading residual part. Register Div = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::DIVD : PPC::DIVW), Div) - .addReg(NegSizeReg) + .addReg(ActualNegSizeReg) .addReg(ScratchReg); Register Mul = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::MULLD : PPC::MULLW), Mul) @@ -11991,7 +11260,7 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI, Register NegMod = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC); BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::SUBF8 : PPC::SUBF), NegMod) .addReg(Mul) - .addReg(NegSizeReg); + .addReg(ActualNegSizeReg); BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg) .addReg(FramePointer) .addReg(SPReg) @@ -12102,9 +11371,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } else if (MI.getOpcode() == PPC::SELECT_CC_F4 || MI.getOpcode() == PPC::SELECT_CC_F8 || MI.getOpcode() == PPC::SELECT_CC_F16 || - MI.getOpcode() == PPC::SELECT_CC_QFRC || - MI.getOpcode() == PPC::SELECT_CC_QSRC || - MI.getOpcode() == PPC::SELECT_CC_QBRC || MI.getOpcode() == PPC::SELECT_CC_VRRC || MI.getOpcode() == PPC::SELECT_CC_VSFRC || MI.getOpcode() == PPC::SELECT_CC_VSSRC || @@ -12114,9 +11380,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || MI.getOpcode() == PPC::SELECT_F16 || - MI.getOpcode() == PPC::SELECT_QFRC || - MI.getOpcode() == PPC::SELECT_QSRC || - MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_SPE || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_VRRC || @@ -12154,9 +11417,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MI.getOpcode() == PPC::SELECT_F16 || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_SPE || - MI.getOpcode() == PPC::SELECT_QFRC || - MI.getOpcode() == PPC::SELECT_QSRC || - MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_VRRC || MI.getOpcode() == PPC::SELECT_VSFRC || MI.getOpcode() == PPC::SELECT_VSSRC || @@ -12639,11 +11899,20 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); // Set rounding mode to round-to-zero. - BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); - BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); + BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)) + .addImm(31) + .addReg(PPC::RM, RegState::ImplicitDefine); + + BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)) + .addImm(30) + .addReg(PPC::RM, RegState::ImplicitDefine); // Perform addition. - BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); + auto MIB = BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest) + .addReg(Src1) + .addReg(Src2); + if (MI.getFlag(MachineInstr::NoFPExcept)) + MIB.setMIFlag(MachineInstr::NoFPExcept); // Restore FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); @@ -12702,10 +11971,12 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, // the immediate to set the bits 62:63 of FPSCR. unsigned Mode = MI.getOperand(1).getImm(); BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) - .addImm(31); + .addImm(31) + .addReg(PPC::RM, RegState::ImplicitDefine); BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) - .addImm(30); + .addImm(30) + .addReg(PPC::RM, RegState::ImplicitDefine); } else if (MI.getOpcode() == PPC::SETRND) { DebugLoc dl = MI.getDebugLoc(); @@ -12815,6 +12086,20 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, .addReg(NewFPSCRReg) .addImm(0) .addImm(0); + } else if (MI.getOpcode() == PPC::SETFLM) { + DebugLoc Dl = MI.getDebugLoc(); + + // Result of setflm is previous FPSCR content, so we need to save it first. + Register OldFPSCRReg = MI.getOperand(0).getReg(); + BuildMI(*BB, MI, Dl, TII->get(PPC::MFFS), OldFPSCRReg); + + // Put bits in 32:63 to FPSCR. + Register NewFPSCRReg = MI.getOperand(1).getReg(); + BuildMI(*BB, MI, Dl, TII->get(PPC::MTFSF)) + .addImm(255) + .addReg(NewFPSCRReg) + .addImm(0) + .addImm(0); } else if (MI.getOpcode() == PPC::PROBED_ALLOCA_32 || MI.getOpcode() == PPC::PROBED_ALLOCA_64) { return emitProbedAlloca(MI, BB); @@ -12841,6 +12126,47 @@ static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { return RefinementSteps; } +SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG, + const DenormalMode &Mode) const { + // We only have VSX Vector Test for software Square Root. + EVT VT = Op.getValueType(); + if (!isTypeLegal(MVT::i1) || + (VT != MVT::f64 && + ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))) + return TargetLowering::getSqrtInputTest(Op, DAG, Mode); + + SDLoc DL(Op); + // The output register of FTSQRT is CR field. + SDValue FTSQRT = DAG.getNode(PPCISD::FTSQRT, DL, MVT::i32, Op); + // ftsqrt BF,FRB + // Let e_b be the unbiased exponent of the double-precision + // floating-point operand in register FRB. + // fe_flag is set to 1 if either of the following conditions occurs. + // - The double-precision floating-point operand in register FRB is a zero, + // a NaN, or an infinity, or a negative value. + // - e_b is less than or equal to -970. + // Otherwise fe_flag is set to 0. + // Both VSX and non-VSX versions would set EQ bit in the CR if the number is + // not eligible for iteration. (zero/negative/infinity/nan or unbiased + // exponent is less than -970) + SDValue SRIdxVal = DAG.getTargetConstant(PPC::sub_eq, DL, MVT::i32); + return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::i1, + FTSQRT, SRIdxVal), + 0); +} + +SDValue +PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op, + SelectionDAG &DAG) const { + // We only have VSX Vector Square Root. + EVT VT = Op.getValueType(); + if (VT != MVT::f64 && + ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())) + return TargetLowering::getSqrtResultForDenormInput(Op, DAG); + + return DAG.getNode(PPCISD::FSQRT, SDLoc(Op), VT, Op); +} + SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, @@ -12849,9 +12175,7 @@ SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || - (VT == MVT::v2f64 && Subtarget.hasVSX()) || - (VT == MVT::v4f32 && Subtarget.hasQPX()) || - (VT == MVT::v4f64 && Subtarget.hasQPX())) { + (VT == MVT::v2f64 && Subtarget.hasVSX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); @@ -12870,9 +12194,7 @@ SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, if ((VT == MVT::f32 && Subtarget.hasFRES()) || (VT == MVT::f64 && Subtarget.hasFRE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || - (VT == MVT::v2f64 && Subtarget.hasVSX()) || - (VT == MVT::v4f32 && Subtarget.hasQPX()) || - (VT == MVT::v4f64 && Subtarget.hasQPX())) { + (VT == MVT::v2f64 && Subtarget.hasVSX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); @@ -12970,24 +12292,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, EVT VT; switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { default: return false; - case Intrinsic::ppc_qpx_qvlfd: - case Intrinsic::ppc_qpx_qvlfda: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvlfs: - case Intrinsic::ppc_qpx_qvlfsa: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvlfcd: - case Intrinsic::ppc_qpx_qvlfcda: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvlfcs: - case Intrinsic::ppc_qpx_qvlfcsa: - VT = MVT::v2f32; - break; - case Intrinsic::ppc_qpx_qvlfiwa: - case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_vsx_lxvw4x: @@ -13016,24 +12320,6 @@ static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, EVT VT; switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { default: return false; - case Intrinsic::ppc_qpx_qvstfd: - case Intrinsic::ppc_qpx_qvstfda: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvstfs: - case Intrinsic::ppc_qpx_qvstfsa: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvstfcd: - case Intrinsic::ppc_qpx_qvstfcda: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvstfcs: - case Intrinsic::ppc_qpx_qvstfcsa: - VT = MVT::v2f32; - break; - case Intrinsic::ppc_qpx_qvstfiw: - case Intrinsic::ppc_qpx_qvstfiwa: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_vsx_stxvw4x: @@ -13261,11 +12547,13 @@ SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); // We don't really care about what is known about the first bit (if - // anything), so clear it in all masks prior to comparing them. - Op1Known.Zero.clearBit(0); Op1Known.One.clearBit(0); - Op2Known.Zero.clearBit(0); Op2Known.One.clearBit(0); + // anything), so pretend that it is known zero for both to ensure they can + // be compared as constants. + Op1Known.Zero.setBit(0); Op1Known.One.clearBit(0); + Op2Known.Zero.setBit(0); Op2Known.One.clearBit(0); - if (Op1Known.Zero != Op2Known.Zero || Op1Known.One != Op2Known.One) + if (!Op1Known.isConstant() || !Op2Known.isConstant() || + Op1Known.getConstant() != Op2Known.getConstant()) return SDValue(); } } @@ -13317,8 +12605,7 @@ SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by extensions. while (!BinOps.empty()) { - SDValue BinOp = BinOps.back(); - BinOps.pop_back(); + SDValue BinOp = BinOps.pop_back_val(); if (!Visited.insert(BinOp.getNode()).second) continue; @@ -13533,8 +12820,7 @@ SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by truncations. while (!BinOps.empty()) { - SDValue BinOp = BinOps.back(); - BinOps.pop_back(); + SDValue BinOp = BinOps.pop_back_val(); if (!Visited.insert(BinOp.getNode()).second) continue; @@ -14131,6 +13417,46 @@ static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { return SDValue(); } +// Look for the pattern of a load from a narrow width to i128, feeding +// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node +// (LXVRZX). This node represents a zero extending load that will be matched +// to the Load VSX Vector Rightmost instructions. +static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) { + SDLoc DL(N); + + // This combine is only eligible for a BUILD_VECTOR of v1i128. + if (N->getValueType(0) != MVT::v1i128) + return SDValue(); + + SDValue Operand = N->getOperand(0); + // Proceed with the transformation if the operand to the BUILD_VECTOR + // is a load instruction. + if (Operand.getOpcode() != ISD::LOAD) + return SDValue(); + + LoadSDNode *LD = dyn_cast<LoadSDNode>(Operand); + EVT MemoryType = LD->getMemoryVT(); + + // This transformation is only valid if the we are loading either a byte, + // halfword, word, or doubleword. + bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 || + MemoryType == MVT::i32 || MemoryType == MVT::i64; + + // Ensure that the load from the narrow width is being zero extended to i128. + if (!ValidLDType || + (LD->getExtensionType() != ISD::ZEXTLOAD && + LD->getExtensionType() != ISD::EXTLOAD)) + return SDValue(); + + SDValue LoadOps[] = { + LD->getChain(), LD->getBasePtr(), + DAG.getIntPtrConstant(MemoryType.getScalarSizeInBits(), DL)}; + + return DAG.getMemIntrinsicNode(PPCISD::LXVRZX, DL, + DAG.getVTList(MVT::v1i128, MVT::Other), + LoadOps, MemoryType, LD->getMemOperand()); +} + SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && @@ -14168,6 +13494,14 @@ SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, return Reduced; } + // On Power10, the Load VSX Vector Rightmost instructions can be utilized + // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR + // is a load from <valid narrow width> to i128. + if (Subtarget.isISA3_1()) { + SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG); + if (BVOfZLoad) + return BVOfZLoad; + } if (N->getValueType(0) != MVT::v2f64) return SDValue(); @@ -14231,6 +13565,8 @@ SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, // from the hardware. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); + if (!Op.getOperand(0).getValueType().isSimple()) + return SDValue(); if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) return SDValue(); @@ -14467,8 +13803,7 @@ SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, EVT Op1VT = N->getOperand(1).getValueType(); EVT ResVT = Val.getValueType(); - // Floating point types smaller than 32 bits are not legal on Power. - if (ResVT.getScalarSizeInBits() < 32) + if (!isTypeLegal(ResVT)) return SDValue(); // Only perform combine for conversion to i64/i32 or power9 i16/i8. @@ -14562,7 +13897,6 @@ static void fixupShuffleMaskForPermutedSToV(SmallVectorImpl<int> &ShuffV, if ((Idx >= 0 && Idx < LHSMaxIdx) || (Idx >= RHSMinIdx && Idx < RHSMaxIdx)) ShuffV[i] += HalfVec; } - return; } // Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if @@ -15031,18 +14365,14 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, EVT MemVT = LD->getMemoryVT(); Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty); - Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); - Align ScalarABIAlignment = DAG.getDataLayout().getABITypeAlign(STy); if (LD->isUnindexed() && VT.isVector() && ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && // P8 and later hardware should just use LOAD. !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || - VT == MVT::v4f32)) || - (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && - LD->getAlign() >= ScalarABIAlignment)) && + VT == MVT::v4f32))) && LD->getAlign() < ABIAlignment) { - // This is a type-legal unaligned Altivec or QPX load. + // This is a type-legal unaligned Altivec load. SDValue Chain = LD->getChain(); SDValue Ptr = LD->getBasePtr(); bool isLittleEndian = Subtarget.isLittleEndian(); @@ -15073,24 +14403,13 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, // optimization later. Intrinsic::ID Intr, IntrLD, IntrPerm; MVT PermCntlTy, PermTy, LDTy; - if (Subtarget.hasAltivec()) { - Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : - Intrinsic::ppc_altivec_lvsl; - IntrLD = Intrinsic::ppc_altivec_lvx; - IntrPerm = Intrinsic::ppc_altivec_vperm; - PermCntlTy = MVT::v16i8; - PermTy = MVT::v4i32; - LDTy = MVT::v4i32; - } else { - Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : - Intrinsic::ppc_qpx_qvlpcls; - IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : - Intrinsic::ppc_qpx_qvlfs; - IntrPerm = Intrinsic::ppc_qpx_qvfperm; - PermCntlTy = MVT::v4f64; - PermTy = MVT::v4f64; - LDTy = MemVT.getSimpleVT(); - } + Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr + : Intrinsic::ppc_altivec_lvsl; + IntrLD = Intrinsic::ppc_altivec_lvx; + IntrPerm = Intrinsic::ppc_altivec_vperm; + PermCntlTy = MVT::v16i8; + PermTy = MVT::v4i32; + LDTy = MVT::v4i32; SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); @@ -15161,10 +14480,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, BaseLoad, ExtraLoad, PermCntl, DAG, dl); if (VT != PermTy) - Perm = Subtarget.hasAltivec() ? - DAG.getNode(ISD::BITCAST, dl, VT, Perm) : - DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX - DAG.getTargetConstant(1, dl, MVT::i64)); + Perm = Subtarget.hasAltivec() + ? DAG.getNode(ISD::BITCAST, dl, VT, Perm) + : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, + DAG.getTargetConstant(1, dl, MVT::i64)); // second argument is 1 because this rounding // is always exact. @@ -15180,14 +14499,10 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr : Intrinsic::ppc_altivec_lvsl); - if ((IID == Intr || - IID == Intrinsic::ppc_qpx_qvlpcld || - IID == Intrinsic::ppc_qpx_qvlpcls) && - N->getOperand(1)->getOpcode() == ISD::ADD) { + if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) { SDValue Add = N->getOperand(1); - int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? - 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; + int Bits = 4 /* 16 byte alignment */; if (DAG.MaskedValueIsZero(Add->getOperand(1), APInt::getAllOnesValue(Bits /* alignment */) @@ -15197,7 +14512,8 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, UE = BasePtr->use_end(); UI != UE; ++UI) { if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && - cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) { + cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == + IID) { // We've found another LVSL/LVSR, and this address is an aligned // multiple of that one. The results will be the same, so use the // one we've just found instead. @@ -15329,43 +14645,43 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, } break; case PPCISD::VCMP: - // If a VCMPo node already exists with exactly the same operands as this - // node, use its result instead of this node (VCMPo computes both a CR6 and - // a normal output). + // If a VCMP_rec node already exists with exactly the same operands as this + // node, use its result instead of this node (VCMP_rec computes both a CR6 + // and a normal output). // if (!N->getOperand(0).hasOneUse() && !N->getOperand(1).hasOneUse() && !N->getOperand(2).hasOneUse()) { - // Scan all of the users of the LHS, looking for VCMPo's that match. - SDNode *VCMPoNode = nullptr; + // Scan all of the users of the LHS, looking for VCMP_rec's that match. + SDNode *VCMPrecNode = nullptr; SDNode *LHSN = N->getOperand(0).getNode(); for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); UI != E; ++UI) - if (UI->getOpcode() == PPCISD::VCMPo && + if (UI->getOpcode() == PPCISD::VCMP_rec && UI->getOperand(1) == N->getOperand(1) && UI->getOperand(2) == N->getOperand(2) && UI->getOperand(0) == N->getOperand(0)) { - VCMPoNode = *UI; + VCMPrecNode = *UI; break; } - // If there is no VCMPo node, or if the flag value has a single use, don't - // transform this. - if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) + // If there is no VCMP_rec node, or if the flag value has a single use, + // don't transform this. + if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(0, 1)) break; // Look at the (necessarily single) use of the flag value. If it has a // chain, this transformation is more complex. Note that multiple things // could use the value result, which we should ignore. SDNode *FlagUser = nullptr; - for (SDNode::use_iterator UI = VCMPoNode->use_begin(); + for (SDNode::use_iterator UI = VCMPrecNode->use_begin(); FlagUser == nullptr; ++UI) { - assert(UI != VCMPoNode->use_end() && "Didn't find user!"); + assert(UI != VCMPrecNode->use_end() && "Didn't find user!"); SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { - if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { + if (User->getOperand(i) == SDValue(VCMPrecNode, 1)) { FlagUser = User; break; } @@ -15375,7 +14691,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, // If the user is a MFOCRF instruction, we know this is safe. // Otherwise we give up for right now. if (FlagUser->getOpcode() == PPCISD::MFOCRF) - return SDValue(VCMPoNode, 0); + return SDValue(VCMPrecNode, 0); } break; case ISD::BRCOND: { @@ -15464,7 +14780,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; - SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); + SDValue CompNode = DAG.getNode(PPCISD::VCMP_rec, dl, VTs, Ops); // Unpack the result based on how the target uses it. PPC::Predicate CompOpc; @@ -15559,16 +14875,19 @@ void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, case Intrinsic::ppc_altivec_vcmpequh_p: case Intrinsic::ppc_altivec_vcmpequw_p: case Intrinsic::ppc_altivec_vcmpequd_p: + case Intrinsic::ppc_altivec_vcmpequq_p: case Intrinsic::ppc_altivec_vcmpgefp_p: case Intrinsic::ppc_altivec_vcmpgtfp_p: case Intrinsic::ppc_altivec_vcmpgtsb_p: case Intrinsic::ppc_altivec_vcmpgtsh_p: case Intrinsic::ppc_altivec_vcmpgtsw_p: case Intrinsic::ppc_altivec_vcmpgtsd_p: + case Intrinsic::ppc_altivec_vcmpgtsq_p: case Intrinsic::ppc_altivec_vcmpgtub_p: case Intrinsic::ppc_altivec_vcmpgtuh_p: case Intrinsic::ppc_altivec_vcmpgtuw_p: case Intrinsic::ppc_altivec_vcmpgtud_p: + case Intrinsic::ppc_altivec_vcmpgtuq_p: Known.Zero = ~1U; // All bits but the low one are known to be zero. break; } @@ -15746,17 +15065,9 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, return std::make_pair(0U, &PPC::F4RCRegClass); if (VT == MVT::f64 || VT == MVT::i64) return std::make_pair(0U, &PPC::F8RCRegClass); - if (VT == MVT::v4f64 && Subtarget.hasQPX()) - return std::make_pair(0U, &PPC::QFRCRegClass); - if (VT == MVT::v4f32 && Subtarget.hasQPX()) - return std::make_pair(0U, &PPC::QSRCRegClass); } break; case 'v': - if (VT == MVT::v4f64 && Subtarget.hasQPX()) - return std::make_pair(0U, &PPC::QFRCRegClass); - if (VT == MVT::v4f32 && Subtarget.hasQPX()) - return std::make_pair(0U, &PPC::QSRCRegClass); if (Subtarget.hasAltivec()) return std::make_pair(0U, &PPC::VRRCRegClass); break; @@ -15892,9 +15203,15 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, // by AM is legal for this target, for a load/store of the specified type. bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, - unsigned AS, Instruction *I) const { - // PPC does not allow r+i addressing modes for vectors! - if (Ty->isVectorTy() && AM.BaseOffs != 0) + unsigned AS, + Instruction *I) const { + // Vector type r+i form is supported since power9 as DQ form. We don't check + // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC, + // imm form is preferred and the offset can be adjusted to use imm form later + // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and + // max offset to check legal addressing mode, we should be a little aggressive + // to contain other offsets for that LSRUse. + if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector()) return false; // PPC allows a sign-extended 16-bit immediate field. @@ -16048,19 +15365,17 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { - case Intrinsic::ppc_qpx_qvlfd: - case Intrinsic::ppc_qpx_qvlfs: - case Intrinsic::ppc_qpx_qvlfcd: - case Intrinsic::ppc_qpx_qvlfcs: - case Intrinsic::ppc_qpx_qvlfiwa: - case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_altivec_lvebx: case Intrinsic::ppc_altivec_lvehx: case Intrinsic::ppc_altivec_lvewx: case Intrinsic::ppc_vsx_lxvd2x: - case Intrinsic::ppc_vsx_lxvw4x: { + case Intrinsic::ppc_vsx_lxvw4x: + case Intrinsic::ppc_vsx_lxvd2x_be: + case Intrinsic::ppc_vsx_lxvw4x_be: + case Intrinsic::ppc_vsx_lxvl: + case Intrinsic::ppc_vsx_lxvll: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_lvebx: @@ -16073,20 +15388,9 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, VT = MVT::i32; break; case Intrinsic::ppc_vsx_lxvd2x: + case Intrinsic::ppc_vsx_lxvd2x_be: VT = MVT::v2f64; break; - case Intrinsic::ppc_qpx_qvlfd: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvlfs: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvlfcd: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvlfcs: - VT = MVT::v2f32; - break; default: VT = MVT::v4i32; break; @@ -16101,52 +15405,17 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags = MachineMemOperand::MOLoad; return true; } - case Intrinsic::ppc_qpx_qvlfda: - case Intrinsic::ppc_qpx_qvlfsa: - case Intrinsic::ppc_qpx_qvlfcda: - case Intrinsic::ppc_qpx_qvlfcsa: - case Intrinsic::ppc_qpx_qvlfiwaa: - case Intrinsic::ppc_qpx_qvlfiwza: { - EVT VT; - switch (Intrinsic) { - case Intrinsic::ppc_qpx_qvlfda: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvlfsa: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvlfcda: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvlfcsa: - VT = MVT::v2f32; - break; - default: - VT = MVT::v4i32; - break; - } - - Info.opc = ISD::INTRINSIC_W_CHAIN; - Info.memVT = VT; - Info.ptrVal = I.getArgOperand(0); - Info.offset = 0; - Info.size = VT.getStoreSize(); - Info.align = Align(1); - Info.flags = MachineMemOperand::MOLoad; - return true; - } - case Intrinsic::ppc_qpx_qvstfd: - case Intrinsic::ppc_qpx_qvstfs: - case Intrinsic::ppc_qpx_qvstfcd: - case Intrinsic::ppc_qpx_qvstfcs: - case Intrinsic::ppc_qpx_qvstfiw: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_altivec_stvebx: case Intrinsic::ppc_altivec_stvehx: case Intrinsic::ppc_altivec_stvewx: case Intrinsic::ppc_vsx_stxvd2x: - case Intrinsic::ppc_vsx_stxvw4x: { + case Intrinsic::ppc_vsx_stxvw4x: + case Intrinsic::ppc_vsx_stxvd2x_be: + case Intrinsic::ppc_vsx_stxvw4x_be: + case Intrinsic::ppc_vsx_stxvl: + case Intrinsic::ppc_vsx_stxvll: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_stvebx: @@ -16159,20 +15428,9 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, VT = MVT::i32; break; case Intrinsic::ppc_vsx_stxvd2x: + case Intrinsic::ppc_vsx_stxvd2x_be: VT = MVT::v2f64; break; - case Intrinsic::ppc_qpx_qvstfd: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvstfs: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvstfcd: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvstfcs: - VT = MVT::v2f32; - break; default: VT = MVT::v4i32; break; @@ -16187,39 +15445,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags = MachineMemOperand::MOStore; return true; } - case Intrinsic::ppc_qpx_qvstfda: - case Intrinsic::ppc_qpx_qvstfsa: - case Intrinsic::ppc_qpx_qvstfcda: - case Intrinsic::ppc_qpx_qvstfcsa: - case Intrinsic::ppc_qpx_qvstfiwa: { - EVT VT; - switch (Intrinsic) { - case Intrinsic::ppc_qpx_qvstfda: - VT = MVT::v4f64; - break; - case Intrinsic::ppc_qpx_qvstfsa: - VT = MVT::v4f32; - break; - case Intrinsic::ppc_qpx_qvstfcda: - VT = MVT::v2f64; - break; - case Intrinsic::ppc_qpx_qvstfcsa: - VT = MVT::v2f32; - break; - default: - VT = MVT::v4i32; - break; - } - - Info.opc = ISD::INTRINSIC_VOID; - Info.memVT = VT; - Info.ptrVal = I.getArgOperand(1); - Info.offset = 0; - Info.size = VT.getStoreSize(); - Info.align = Align(1); - Info.flags = MachineMemOperand::MOStore; - return true; - } default: break; } @@ -16232,14 +15457,6 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, EVT PPCTargetLowering::getOptimalMemOpType( const MemOp &Op, const AttributeList &FuncAttributes) const { if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { - // When expanding a memset, require at least two QPX instructions to cover - // the cost of loading the value to be stored from the constant pool. - if (Subtarget.hasQPX() && Op.size() >= 32 && - (Op.isMemcpy() || Op.size() >= 64) && Op.isAligned(Align(32)) && - !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { - return MVT::v4f64; - } - // We should use Altivec/VSX loads and stores when available. For unaligned // addresses, unaligned VSX loads are only fast starting with the P8. if (Subtarget.hasAltivec() && Op.size() >= 16 && @@ -16358,6 +15575,33 @@ bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, return true; } +bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, + SDValue C) const { + // Check integral scalar types. + if (!VT.isScalarInteger()) + return false; + if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) { + if (!ConstNode->getAPIntValue().isSignedIntN(64)) + return false; + // This transformation will generate >= 2 operations. But the following + // cases will generate <= 2 instructions during ISEL. So exclude them. + // 1. If the constant multiplier fits 16 bits, it can be handled by one + // HW instruction, ie. MULLI + // 2. If the multiplier after shifted fits 16 bits, an extra shift + // instruction is needed than case 1, ie. MULLI and RLDICR + int64_t Imm = ConstNode->getSExtValue(); + unsigned Shift = countTrailingZeros<uint64_t>(Imm); + Imm >>= Shift; + if (isInt<16>(Imm)) + return false; + uint64_t UImm = static_cast<uint64_t>(Imm); + if (isPowerOf2_64(UImm + 1) || isPowerOf2_64(UImm - 1) || + isPowerOf2_64(1 - UImm) || isPowerOf2_64(-1 - UImm)) + return true; + } + return false; +} + bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const { return isFMAFasterThanFMulAndFAdd( @@ -16377,31 +15621,56 @@ bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F, } } -// Currently this is a copy from AArch64TargetLowering::isProfitableToHoist. -// FIXME: add more patterns which are profitable to hoist. +// FIXME: add more patterns which are not profitable to hoist. bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const { - if (I->getOpcode() != Instruction::FMul) - return true; - if (!I->hasOneUse()) return true; Instruction *User = I->user_back(); assert(User && "A single use instruction with no uses."); - if (User->getOpcode() != Instruction::FSub && - User->getOpcode() != Instruction::FAdd) - return true; + switch (I->getOpcode()) { + case Instruction::FMul: { + // Don't break FMA, PowerPC prefers FMA. + if (User->getOpcode() != Instruction::FSub && + User->getOpcode() != Instruction::FAdd) + return true; - const TargetOptions &Options = getTargetMachine().Options; - const Function *F = I->getFunction(); - const DataLayout &DL = F->getParent()->getDataLayout(); - Type *Ty = User->getOperand(0)->getType(); + const TargetOptions &Options = getTargetMachine().Options; + const Function *F = I->getFunction(); + const DataLayout &DL = F->getParent()->getDataLayout(); + Type *Ty = User->getOperand(0)->getType(); + + return !( + isFMAFasterThanFMulAndFAdd(*F, Ty) && + isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) && + (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath)); + } + case Instruction::Load: { + // Don't break "store (load float*)" pattern, this pattern will be combined + // to "store (load int32)" in later InstCombine pass. See function + // combineLoadToOperationType. On PowerPC, loading a float point takes more + // cycles than loading a 32 bit integer. + LoadInst *LI = cast<LoadInst>(I); + // For the loads that combineLoadToOperationType does nothing, like + // ordered load, it should be profitable to hoist them. + // For swifterror load, it can only be used for pointer to pointer type, so + // later type check should get rid of this case. + if (!LI->isUnordered()) + return true; + + if (User->getOpcode() != Instruction::Store) + return true; + + if (I->getType()->getTypeID() != Type::FloatTyID) + return true; - return !( - isFMAFasterThanFMulAndFAdd(*F, Ty) && - isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) && - (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath)); + return false; + } + default: + return true; + } + return true; } const MCPhysReg * @@ -16433,7 +15702,7 @@ PPCTargetLowering::shouldExpandBuildVectorWithShuffles( if (VT == MVT::v2i64) return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves - if (Subtarget.hasVSX() || Subtarget.hasQPX()) + if (Subtarget.hasVSX()) return true; return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); @@ -16479,8 +15748,7 @@ SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, switch (Opc) { case PPCISD::FNMSUB: - // TODO: QPX subtarget is deprecated. No transformation here. - if (!Op.hasOneUse() || !isTypeLegal(VT) || Subtarget.hasQPX()) + if (!Op.hasOneUse() || !isTypeLegal(VT)) break; const TargetOptions &Options = getTargetMachine().Options; @@ -16609,10 +15877,10 @@ SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { SDValue N0 = N->getOperand(0); ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1)); - if (!Subtarget.isISA3_0() || + if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() || N0.getOpcode() != ISD::SIGN_EXTEND || - N0.getOperand(0).getValueType() != MVT::i32 || - CN1 == nullptr || N->getValueType(0) != MVT::i64) + N0.getOperand(0).getValueType() != MVT::i32 || CN1 == nullptr || + N->getValueType(0) != MVT::i64) return SDValue(); // We can't save an operation here if the value is already extended, and @@ -16961,8 +16229,7 @@ SDValue PPCTargetLowering::combineFMALike(SDNode *N, bool LegalOps = !DCI.isBeforeLegalizeOps(); SDLoc Loc(N); - // TODO: QPX subtarget is deprecated. No transformation here. - if (Subtarget.hasQPX() || !isOperationLegal(ISD::FMA, VT)) + if (!isOperationLegal(ISD::FMA, VT)) return SDValue(); // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0 |