diff options
Diffstat (limited to 'llvm/lib/Target/PowerPC/PPCISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 1558 |
1 files changed, 1368 insertions, 190 deletions
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 9215c17cb94b..37358176f35e 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -121,10 +121,10 @@ cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables", cl::desc("use absolute jump tables on ppc"), cl::Hidden); -// TODO - Remove this option if soft fp128 has been fully supported . -static cl::opt<bool> - EnableSoftFP128("enable-soft-fp128", - cl::desc("temp option to enable soft fp128"), cl::Hidden); +static cl::opt<bool> EnableQuadwordAtomics( + "ppc-quadword-atomics", + cl::desc("enable quadword lock-free atomic operations"), cl::init(false), + cl::Hidden); STATISTIC(NumTailCalls, "Number of tail calls"); STATISTIC(NumSiblingCalls, "Number of sibling calls"); @@ -135,12 +135,18 @@ static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); +static const char AIXSSPCanaryWordName[] = "__ssp_canary_word"; + // FIXME: Remove this once the bug has been fixed! extern cl::opt<bool> ANDIGlueBug; PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, const PPCSubtarget &STI) : TargetLowering(TM), Subtarget(STI) { + // Initialize map that relates the PPC addressing modes to the computed flags + // of a load/store instruction. The map is used to determine the optimal + // addressing mode when selecting load and stores. + initializeAddrModeMap(); // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all // arguments are at least 4/8 bytes aligned. bool isPPC64 = Subtarget.isPPC64(); @@ -167,6 +173,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); + // Custom lower inline assembly to check for special registers. + setOperationAction(ISD::INLINEASM, MVT::Other, Custom); + setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); + // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); @@ -324,14 +334,18 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal); - setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal); setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal); - setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal); + + if (!Subtarget.hasSPE()) { + setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal); + } + if (Subtarget.hasVSX()) { setOperationAction(ISD::STRICT_FRINT, MVT::f32, Legal); setOperationAction(ISD::STRICT_FRINT, MVT::f64, Legal); @@ -413,7 +427,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // to speed up scalar BSWAP64. // CTPOP or CTTZ were introduced in P8/P9 respectively setOperationAction(ISD::BSWAP, MVT::i32 , Expand); - if (Subtarget.hasP9Vector()) + if (Subtarget.hasP9Vector() && Subtarget.isPPC64()) setOperationAction(ISD::BSWAP, MVT::i64 , Custom); else setOperationAction(ISD::BSWAP, MVT::i64 , Expand); @@ -477,6 +491,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); + + // SPE supports signaling compare of f32/f64. + setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal); } else { // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); @@ -878,6 +896,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, if (Subtarget.hasVSX()) { setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); + setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); } if (Subtarget.hasP8Altivec()) @@ -1065,8 +1084,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); - if (Subtarget.hasDirectMove()) - setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); + setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); // Handle constrained floating-point operations of vector. @@ -1101,6 +1119,23 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal); addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); + addRegisterClass(MVT::f128, &PPC::VRRCRegClass); + + for (MVT FPT : MVT::fp_valuetypes()) + setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); + + // Expand the SELECT to SELECT_CC + setOperationAction(ISD::SELECT, MVT::f128, Expand); + + setTruncStoreAction(MVT::f128, MVT::f64, Expand); + setTruncStoreAction(MVT::f128, MVT::f32, Expand); + + // No implementation for these ops for PowerPC. + setOperationAction(ISD::FSIN, MVT::f128, Expand); + setOperationAction(ISD::FCOS, MVT::f128, Expand); + setOperationAction(ISD::FPOW, MVT::f128, Expand); + setOperationAction(ISD::FPOWI, MVT::f128, Expand); + setOperationAction(ISD::FREM, MVT::f128, Expand); } if (Subtarget.hasP8Altivec()) { @@ -1119,15 +1154,12 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SRL, MVT::v1i128, Legal); setOperationAction(ISD::SRA, MVT::v1i128, Expand); - addRegisterClass(MVT::f128, &PPC::VRRCRegClass); setOperationAction(ISD::FADD, MVT::f128, Legal); setOperationAction(ISD::FSUB, MVT::f128, Legal); setOperationAction(ISD::FDIV, MVT::f128, Legal); setOperationAction(ISD::FMUL, MVT::f128, Legal); setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); - // No extending loads to f128 on PPC. - for (MVT FPT : MVT::fp_valuetypes()) - setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); + setOperationAction(ISD::FMA, MVT::f128, Legal); setCondCodeAction(ISD::SETULT, MVT::f128, Expand); setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); @@ -1143,18 +1175,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); setOperationAction(ISD::FROUND, MVT::f128, Legal); - setOperationAction(ISD::SELECT, MVT::f128, Expand); setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); - setTruncStoreAction(MVT::f128, MVT::f64, Expand); - setTruncStoreAction(MVT::f128, MVT::f32, Expand); setOperationAction(ISD::BITCAST, MVT::i128, Custom); - // No implementation for these ops for PowerPC. - setOperationAction(ISD::FSIN, MVT::f128, Expand); - setOperationAction(ISD::FCOS, MVT::f128, Expand); - setOperationAction(ISD::FPOW, MVT::f128, Expand); - setOperationAction(ISD::FPOWI, MVT::f128, Expand); - setOperationAction(ISD::FREM, MVT::f128, Expand); // Handle constrained floating-point operations of fp128 setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal); @@ -1177,12 +1200,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::BSWAP, MVT::v4i32, Legal); setOperationAction(ISD::BSWAP, MVT::v2i64, Legal); setOperationAction(ISD::BSWAP, MVT::v1i128, Legal); - } else if (Subtarget.hasAltivec() && EnableSoftFP128) { - addRegisterClass(MVT::f128, &PPC::VRRCRegClass); - - for (MVT FPT : MVT::fp_valuetypes()) - setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); - + } else if (Subtarget.hasVSX()) { setOperationAction(ISD::LOAD, MVT::f128, Promote); setOperationAction(ISD::STORE, MVT::f128, Promote); @@ -1198,18 +1216,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FDIV, MVT::f128, Expand); setOperationAction(ISD::FNEG, MVT::f128, Expand); setOperationAction(ISD::FABS, MVT::f128, Expand); - setOperationAction(ISD::FSIN, MVT::f128, Expand); - setOperationAction(ISD::FCOS, MVT::f128, Expand); - setOperationAction(ISD::FPOW, MVT::f128, Expand); - setOperationAction(ISD::FPOWI, MVT::f128, Expand); - setOperationAction(ISD::FREM, MVT::f128, Expand); setOperationAction(ISD::FSQRT, MVT::f128, Expand); setOperationAction(ISD::FMA, MVT::f128, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); - setTruncStoreAction(MVT::f128, MVT::f64, Expand); - setTruncStoreAction(MVT::f128, MVT::f32, Expand); - // Expand the fp_extend if the target type is fp128. setOperationAction(ISD::FP_EXTEND, MVT::f128, Expand); setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Expand); @@ -1219,6 +1229,19 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::FP_ROUND, VT, Custom); setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom); } + + setOperationAction(ISD::SETCC, MVT::f128, Custom); + setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom); + setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom); + setOperationAction(ISD::BR_CC, MVT::f128, Expand); + + // Lower following f128 select_cc pattern: + // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE + setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); + + // We need to handle f128 SELECT_CC with integer result type. + setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); + setOperationAction(ISD::SELECT_CC, MVT::i64, isPPC64 ? Custom : Expand); } if (Subtarget.hasP9Altivec()) { @@ -1233,6 +1256,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); } + + if (Subtarget.isISA3_1()) + setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); } if (Subtarget.pairedVectorMemops()) { @@ -1260,6 +1286,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); } + if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics()) + setMaxAtomicSizeInBitsSupported(128); + setBooleanContents(ZeroOrOneBooleanContent); if (Subtarget.hasAltivec()) { @@ -1412,6 +1441,84 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive(); } +// *********************************** NOTE ************************************ +// For selecting load and store instructions, the addressing modes are defined +// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD +// patterns to match the load the store instructions. +// +// The TD definitions for the addressing modes correspond to their respective +// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely +// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the +// address mode flags of a particular node. Afterwards, the computed address +// flags are passed into getAddrModeForFlags() in order to retrieve the optimal +// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement +// accordingly, based on the preferred addressing mode. +// +// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode. +// MemOpFlags contains all the possible flags that can be used to compute the +// optimal addressing mode for load and store instructions. +// AddrMode contains all the possible load and store addressing modes available +// on Power (such as DForm, DSForm, DQForm, XForm, etc.) +// +// When adding new load and store instructions, it is possible that new address +// flags may need to be added into MemOpFlags, and a new addressing mode will +// need to be added to AddrMode. An entry of the new addressing mode (consisting +// of the minimal and main distinguishing address flags for the new load/store +// instructions) will need to be added into initializeAddrModeMap() below. +// Finally, when adding new addressing modes, the getAddrModeForFlags() will +// need to be updated to account for selecting the optimal addressing mode. +// ***************************************************************************** +/// Initialize the map that relates the different addressing modes of the load +/// and store instructions to a set of flags. This ensures the load/store +/// instruction is correctly matched during instruction selection. +void PPCTargetLowering::initializeAddrModeMap() { + AddrModesMap[PPC::AM_DForm] = { + // LWZ, STW + PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt, + PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt, + PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt, + PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt, + // LBZ, LHZ, STB, STH + PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt, + PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt, + PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt, + PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt, + // LHA + PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt, + PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt, + PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt, + PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt, + // LFS, LFD, STFS, STFD + PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, + PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, + PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, + PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9, + }; + AddrModesMap[PPC::AM_DSForm] = { + // LWA + PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt, + PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt, + PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt, + // LD, STD + PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt, + PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt, + PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt, + // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64 + PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, + PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, + PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9, + }; + AddrModesMap[PPC::AM_DQForm] = { + // LXV, STXV + PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9, + PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9, + PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9, + PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10, + PPC::MOF_NotAddNorCst | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10, + PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector256 | PPC::MOF_SubtargetP10, + }; +} + /// getMaxByValAlign - Helper for getByValTypeAlignment to determine /// the desired ByVal argument alignment. static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) { @@ -1567,6 +1674,7 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; + case PPCISD::TLSGD_AIX: return "PPCISD::TLSGD_AIX"; case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; @@ -2423,6 +2531,20 @@ bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { return isIntS16Immediate(Op.getNode(), Imm); } +/// Used when computing address flags for selecting loads and stores. +/// If we have an OR, check if the LHS and RHS are provably disjoint. +/// An OR of two provably disjoint values is equivalent to an ADD. +/// Most PPC load/store instructions compute the effective address as a sum, +/// so doing this conversion is useful. +static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) { + if (N.getOpcode() != ISD::OR) + return false; + KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); + if (!LHSKnown.Zero.getBoolValue()) + return false; + KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); + return (~(LHSKnown.Zero | RHSKnown.Zero) == 0); +} /// SelectAddressEVXRegReg - Given the specified address, check to see if it can /// be represented as an indexed [r+r] operation. @@ -3112,6 +3234,41 @@ SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { + if (Subtarget.isAIXABI()) + return LowerGlobalTLSAddressAIX(Op, DAG); + + return LowerGlobalTLSAddressLinux(Op, DAG); +} + +SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op, + SelectionDAG &DAG) const { + GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); + + if (DAG.getTarget().useEmulatedTLS()) + report_fatal_error("Emulated TLS is not yet supported on AIX"); + + SDLoc dl(GA); + const GlobalValue *GV = GA->getGlobal(); + EVT PtrVT = getPointerTy(DAG.getDataLayout()); + + // The general-dynamic model is the only access model supported for now, so + // all the GlobalTLSAddress nodes are lowered with this model. + // We need to generate two TOC entries, one for the variable offset, one for + // the region handle. The global address for the TOC entry of the region + // handle is created with the MO_TLSGDM_FLAG flag and the global address + // for the TOC entry of the variable offset is created with MO_TLSGD_FLAG. + SDValue VariableOffsetTGA = + DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGD_FLAG); + SDValue RegionHandleTGA = + DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGDM_FLAG); + SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA); + SDValue RegionHandle = getTOCEntry(DAG, dl, RegionHandleTGA); + return DAG.getNode(PPCISD::TLSGD_AIX, dl, PtrVT, VariableOffset, + RegionHandle); +} + +SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, + SelectionDAG &DAG) const { // FIXME: TLS addresses currently use medium model code sequences, // which is the most useful form. Eventually support for small and // large models could be added if users need it, at the cost of @@ -3290,21 +3447,43 @@ SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, } SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { - ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); + bool IsStrict = Op->isStrictFPOpcode(); + ISD::CondCode CC = + cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get(); + SDValue LHS = Op.getOperand(IsStrict ? 1 : 0); + SDValue RHS = Op.getOperand(IsStrict ? 2 : 1); + SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); + EVT LHSVT = LHS.getValueType(); SDLoc dl(Op); + // Soften the setcc with libcall if it is fp128. + if (LHSVT == MVT::f128) { + assert(!Subtarget.hasP9Vector() && + "SETCC for f128 is already legal under Power9!"); + softenSetCCOperands(DAG, LHSVT, LHS, RHS, CC, dl, LHS, RHS, Chain, + Op->getOpcode() == ISD::STRICT_FSETCCS); + if (RHS.getNode()) + LHS = DAG.getNode(ISD::SETCC, dl, Op.getValueType(), LHS, RHS, + DAG.getCondCode(CC)); + if (IsStrict) + return DAG.getMergeValues({LHS, Chain}, dl); + return LHS; + } + + assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!"); + if (Op.getValueType() == MVT::v2i64) { // When the operands themselves are v2i64 values, we need to do something // special because VSX has no underlying comparison operations for these. - if (Op.getOperand(0).getValueType() == MVT::v2i64) { + if (LHS.getValueType() == MVT::v2i64) { // Equality can be handled by casting to the legal type for Altivec // comparisons, everything else needs to be expanded. if (CC == ISD::SETEQ || CC == ISD::SETNE) { - return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, - DAG.getSetCC(dl, MVT::v4i32, - DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), - DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), - CC)); + return DAG.getNode( + ISD::BITCAST, dl, MVT::v2i64, + DAG.getSetCC(dl, MVT::v4i32, + DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS), + DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC)); } return SDValue(); @@ -3320,7 +3499,7 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) return V; - if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) { // Leave comparisons against 0 and -1 alone for now, since they're usually // optimized. FIXME: revisit this when we can custom lower all setcc // optimizations. @@ -3333,11 +3512,9 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { // condition register, reading it back out, and masking the correct bit. The // normal approach here uses sub to do this instead of xor. Using xor exposes // the result to other bit-twiddling opportunities. - EVT LHSVT = Op.getOperand(0).getValueType(); if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { EVT VT = Op.getValueType(); - SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), - Op.getOperand(1)); + SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, LHS, RHS); return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); } return SDValue(); @@ -3461,6 +3638,57 @@ SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, return Op.getOperand(0); } +SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>(); + + assert((Op.getOpcode() == ISD::INLINEASM || + Op.getOpcode() == ISD::INLINEASM_BR) && + "Expecting Inline ASM node."); + + // If an LR store is already known to be required then there is not point in + // checking this ASM as well. + if (MFI.isLRStoreRequired()) + return Op; + + // Inline ASM nodes have an optional last operand that is an incoming Flag of + // type MVT::Glue. We want to ignore this last operand if that is the case. + unsigned NumOps = Op.getNumOperands(); + if (Op.getOperand(NumOps - 1).getValueType() == MVT::Glue) + --NumOps; + + // Check all operands that may contain the LR. + for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { + unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue(); + unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); + ++i; // Skip the ID value. + + switch (InlineAsm::getKind(Flags)) { + default: + llvm_unreachable("Bad flags!"); + case InlineAsm::Kind_RegUse: + case InlineAsm::Kind_Imm: + case InlineAsm::Kind_Mem: + i += NumVals; + break; + case InlineAsm::Kind_Clobber: + case InlineAsm::Kind_RegDef: + case InlineAsm::Kind_RegDefEarlyClobber: { + for (; NumVals; --NumVals, ++i) { + Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg(); + if (Reg != PPC::LR && Reg != PPC::LR8) + continue; + MFI.setLRStoreRequired(); + return Op; + } + break; + } + } + } + + return Op; +} + SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const { if (Subtarget.isAIXABI()) @@ -4924,7 +5152,7 @@ static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG, if (isPatchPoint) return false; - if (isFunctionGlobalAddress(Callee) || dyn_cast<ExternalSymbolSDNode>(Callee)) + if (isFunctionGlobalAddress(Callee) || isa<ExternalSymbolSDNode>(Callee)) return false; // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not @@ -5049,8 +5277,8 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG, const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) { auto &Context = DAG.getMachineFunction().getMMI().getContext(); MCSectionXCOFF *Sec = Context.getXCOFFSection( - (Twine(".") + Twine(SymName)).str(), XCOFF::XMC_PR, XCOFF::XTY_ER, - SectionKind::getMetadata()); + (Twine(".") + Twine(SymName)).str(), SectionKind::getMetadata(), + XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER)); return Sec->getQualNameSymbol(); }; @@ -6257,21 +6485,49 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( Callee, SPDiff, NumBytes, Ins, InVals, CB); } +// Returns true when the shadow of a general purpose argument register +// in the parameter save area is aligned to at least 'RequiredAlign'. +static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) { + assert(RequiredAlign.value() <= 16 && + "Required alignment greater than stack alignment."); + switch (Reg) { + default: + report_fatal_error("called on invalid register."); + case PPC::R5: + case PPC::R9: + case PPC::X3: + case PPC::X5: + case PPC::X7: + case PPC::X9: + // These registers are 16 byte aligned which is the most strict aligment + // we can support. + return true; + case PPC::R3: + case PPC::R7: + case PPC::X4: + case PPC::X6: + case PPC::X8: + case PPC::X10: + // The shadow of these registers in the PSA is 8 byte aligned. + return RequiredAlign <= 8; + case PPC::R4: + case PPC::R6: + case PPC::R8: + case PPC::R10: + return RequiredAlign <= 4; + } +} + static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, - CCState &State) { - + CCState &S) { + AIXCCState &State = static_cast<AIXCCState &>(S); const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>( State.getMachineFunction().getSubtarget()); const bool IsPPC64 = Subtarget.isPPC64(); const Align PtrAlign = IsPPC64 ? Align(8) : Align(4); const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; - if (ValVT.isVector() && !State.getMachineFunction() - .getTarget() - .Options.EnableAIXExtendedAltivecABI) - report_fatal_error("the default Altivec AIX ABI is not yet supported"); - if (ValVT == MVT::f128) report_fatal_error("f128 is unimplemented on AIX."); @@ -6392,15 +6648,96 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, case MVT::v2i64: case MVT::v2f64: case MVT::v1i128: { - if (State.isVarArg()) - report_fatal_error( - "variadic arguments for vector types are unimplemented for AIX"); + const unsigned VecSize = 16; + const Align VecAlign(VecSize); - if (unsigned VReg = State.AllocateReg(VR)) - State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); - else { - report_fatal_error( - "passing vector parameters to the stack is unimplemented for AIX"); + if (!State.isVarArg()) { + // If there are vector registers remaining we don't consume any stack + // space. + if (unsigned VReg = State.AllocateReg(VR)) { + State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); + return false; + } + // Vectors passed on the stack do not shadow GPRs or FPRs even though they + // might be allocated in the portion of the PSA that is shadowed by the + // GPRs. + const unsigned Offset = State.AllocateStack(VecSize, VecAlign); + State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); + return false; + } + + const unsigned PtrSize = IsPPC64 ? 8 : 4; + ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32; + + unsigned NextRegIndex = State.getFirstUnallocated(GPRs); + // Burn any underaligned registers and their shadowed stack space until + // we reach the required alignment. + while (NextRegIndex != GPRs.size() && + !isGPRShadowAligned(GPRs[NextRegIndex], VecAlign)) { + // Shadow allocate register and its stack shadow. + unsigned Reg = State.AllocateReg(GPRs); + State.AllocateStack(PtrSize, PtrAlign); + assert(Reg && "Allocating register unexpectedly failed."); + (void)Reg; + NextRegIndex = State.getFirstUnallocated(GPRs); + } + + // Vectors that are passed as fixed arguments are handled differently. + // They are passed in VRs if any are available (unlike arguments passed + // through ellipses) and shadow GPRs (unlike arguments to non-vaarg + // functions) + if (State.isFixed(ValNo)) { + if (unsigned VReg = State.AllocateReg(VR)) { + State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo)); + // Shadow allocate GPRs and stack space even though we pass in a VR. + for (unsigned I = 0; I != VecSize; I += PtrSize) + State.AllocateReg(GPRs); + State.AllocateStack(VecSize, VecAlign); + return false; + } + // No vector registers remain so pass on the stack. + const unsigned Offset = State.AllocateStack(VecSize, VecAlign); + State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); + return false; + } + + // If all GPRS are consumed then we pass the argument fully on the stack. + if (NextRegIndex == GPRs.size()) { + const unsigned Offset = State.AllocateStack(VecSize, VecAlign); + State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); + return false; + } + + // Corner case for 32-bit codegen. We have 2 registers to pass the first + // half of the argument, and then need to pass the remaining half on the + // stack. + if (GPRs[NextRegIndex] == PPC::R9) { + const unsigned Offset = State.AllocateStack(VecSize, VecAlign); + State.addLoc( + CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo)); + + const unsigned FirstReg = State.AllocateReg(PPC::R9); + const unsigned SecondReg = State.AllocateReg(PPC::R10); + assert(FirstReg && SecondReg && + "Allocating R9 or R10 unexpectedly failed."); + State.addLoc( + CCValAssign::getCustomReg(ValNo, ValVT, FirstReg, RegVT, LocInfo)); + State.addLoc( + CCValAssign::getCustomReg(ValNo, ValVT, SecondReg, RegVT, LocInfo)); + return false; + } + + // We have enough GPRs to fully pass the vector argument, and we have + // already consumed any underaligned registers. Start with the custom + // MemLoc and then the custom RegLocs. + const unsigned Offset = State.AllocateStack(VecSize, VecAlign); + State.addLoc( + CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo)); + for (unsigned I = 0; I != VecSize; I += PtrSize) { + const unsigned Reg = State.AllocateReg(GPRs); + assert(Reg && "Failed to allocated register for vararg vector argument"); + State.addLoc( + CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo)); } return false; } @@ -6408,8 +6745,11 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, return true; } +// So far, this function is only used by LowerFormalArguments_AIX() static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, - bool IsPPC64) { + bool IsPPC64, + bool HasP8Vector, + bool HasVSX) { assert((IsPPC64 || SVT != MVT::i64) && "i64 should have been split for 32-bit codegen."); @@ -6421,9 +6761,9 @@ static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, case MVT::i64: return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; case MVT::f32: - return &PPC::F4RCRegClass; + return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass; case MVT::f64: - return &PPC::F8RCRegClass; + return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: @@ -6535,7 +6875,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); - CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); + AIXCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); const EVT PtrVT = getPointerTy(MF.getDataLayout()); // Reserve space for the linkage area on the stack. @@ -6548,27 +6888,116 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) { CCValAssign &VA = ArgLocs[I++]; MVT LocVT = VA.getLocVT(); + MVT ValVT = VA.getValVT(); ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags; - if (VA.isMemLoc() && VA.getValVT().isVector()) - report_fatal_error( - "passing vector parameters to the stack is unimplemented for AIX"); - // For compatibility with the AIX XL compiler, the float args in the // parameter save area are initialized even if the argument is available // in register. The caller is required to initialize both the register // and memory, however, the callee can choose to expect it in either. // The memloc is dismissed here because the argument is retrieved from // the register. - if (VA.isMemLoc() && VA.needsCustom()) + if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint()) + continue; + + auto HandleMemLoc = [&]() { + const unsigned LocSize = LocVT.getStoreSize(); + const unsigned ValSize = ValVT.getStoreSize(); + assert((ValSize <= LocSize) && + "Object size is larger than size of MemLoc"); + int CurArgOffset = VA.getLocMemOffset(); + // Objects are right-justified because AIX is big-endian. + if (LocSize > ValSize) + CurArgOffset += LocSize - ValSize; + // Potential tail calls could cause overwriting of argument stack slots. + const bool IsImmutable = + !(getTargetMachine().Options.GuaranteedTailCallOpt && + (CallConv == CallingConv::Fast)); + int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable); + SDValue FIN = DAG.getFrameIndex(FI, PtrVT); + SDValue ArgValue = + DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo()); + InVals.push_back(ArgValue); + }; + + // Vector arguments to VaArg functions are passed both on the stack, and + // in any available GPRs. Load the value from the stack and add the GPRs + // as live ins. + if (VA.isMemLoc() && VA.needsCustom()) { + assert(ValVT.isVector() && "Unexpected Custom MemLoc type."); + assert(isVarArg && "Only use custom memloc for vararg."); + // ValNo of the custom MemLoc, so we can compare it to the ValNo of the + // matching custom RegLocs. + const unsigned OriginalValNo = VA.getValNo(); + (void)OriginalValNo; + + auto HandleCustomVecRegLoc = [&]() { + assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && + "Missing custom RegLoc."); + VA = ArgLocs[I++]; + assert(VA.getValVT().isVector() && + "Unexpected Val type for custom RegLoc."); + assert(VA.getValNo() == OriginalValNo && + "ValNo mismatch between custom MemLoc and RegLoc."); + MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy; + MF.addLiveIn(VA.getLocReg(), + getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(), + Subtarget.hasVSX())); + }; + + HandleMemLoc(); + // In 64-bit there will be exactly 2 custom RegLocs that follow, and in + // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and + // R10. + HandleCustomVecRegLoc(); + HandleCustomVecRegLoc(); + + // If we are targeting 32-bit, there might be 2 extra custom RegLocs if + // we passed the vector in R5, R6, R7 and R8. + if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) { + assert(!IsPPC64 && + "Only 2 custom RegLocs expected for 64-bit codegen."); + HandleCustomVecRegLoc(); + HandleCustomVecRegLoc(); + } + continue; + } if (VA.isRegLoc()) { if (VA.getValVT().isScalarInteger()) FuncInfo->appendParameterType(PPCFunctionInfo::FixedType); - else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) - FuncInfo->appendParameterType(VA.getValVT().SimpleTy == MVT::f32 - ? PPCFunctionInfo::ShortFloatPoint - : PPCFunctionInfo::LongFloatPoint); + else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) { + switch (VA.getValVT().SimpleTy) { + default: + report_fatal_error("Unhandled value type for argument."); + case MVT::f32: + FuncInfo->appendParameterType(PPCFunctionInfo::ShortFloatingPoint); + break; + case MVT::f64: + FuncInfo->appendParameterType(PPCFunctionInfo::LongFloatingPoint); + break; + } + } else if (VA.getValVT().isVector()) { + switch (VA.getValVT().SimpleTy) { + default: + report_fatal_error("Unhandled value type for argument."); + case MVT::v16i8: + FuncInfo->appendParameterType(PPCFunctionInfo::VectorChar); + break; + case MVT::v8i16: + FuncInfo->appendParameterType(PPCFunctionInfo::VectorShort); + break; + case MVT::v4i32: + case MVT::v2i64: + case MVT::v1i128: + FuncInfo->appendParameterType(PPCFunctionInfo::VectorInt); + break; + case MVT::v4f32: + case MVT::v2f64: + FuncInfo->appendParameterType(PPCFunctionInfo::VectorFloat); + break; + } + } } if (Flags.isByVal() && VA.isMemLoc()) { @@ -6649,11 +7078,12 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( continue; } - EVT ValVT = VA.getValVT(); if (VA.isRegLoc() && !VA.needsCustom()) { - MVT::SimpleValueType SVT = ValVT.getSimpleVT().SimpleTy; - unsigned VReg = - MF.addLiveIn(VA.getLocReg(), getRegClassForSVT(SVT, IsPPC64)); + MVT::SimpleValueType SVT = ValVT.SimpleTy; + Register VReg = + MF.addLiveIn(VA.getLocReg(), + getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(), + Subtarget.hasVSX())); SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); if (ValVT.isScalarInteger() && (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) { @@ -6664,23 +7094,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX( continue; } if (VA.isMemLoc()) { - const unsigned LocSize = LocVT.getStoreSize(); - const unsigned ValSize = ValVT.getStoreSize(); - assert((ValSize <= LocSize) && - "Object size is larger than size of MemLoc"); - int CurArgOffset = VA.getLocMemOffset(); - // Objects are right-justified because AIX is big-endian. - if (LocSize > ValSize) - CurArgOffset += LocSize - ValSize; - // Potential tail calls could cause overwriting of argument stack slots. - const bool IsImmutable = - !(getTargetMachine().Options.GuaranteedTailCallOpt && - (CallConv == CallingConv::Fast)); - int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable); - SDValue FIN = DAG.getFrameIndex(FI, PtrVT); - SDValue ArgValue = - DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo()); - InVals.push_back(ArgValue); + HandleMemLoc(); continue; } } @@ -6761,8 +7175,8 @@ SDValue PPCTargetLowering::LowerCall_AIX( MachineFunction &MF = DAG.getMachineFunction(); SmallVector<CCValAssign, 16> ArgLocs; - CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, - *DAG.getContext()); + AIXCCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, + *DAG.getContext()); // Reserve space for the linkage save area (LSA) on the stack. // In both PPC32 and PPC64 there are 6 reserved slots in the LSA: @@ -6903,10 +7317,6 @@ SDValue PPCTargetLowering::LowerCall_AIX( const MVT LocVT = VA.getLocVT(); const MVT ValVT = VA.getValVT(); - if (VA.isMemLoc() && VA.getValVT().isVector()) - report_fatal_error( - "passing vector parameters to the stack is unimplemented for AIX"); - switch (VA.getLocInfo()) { default: report_fatal_error("Unexpected argument extension type."); @@ -6925,6 +7335,52 @@ SDValue PPCTargetLowering::LowerCall_AIX( continue; } + // Vector arguments passed to VarArg functions need custom handling when + // they are passed (at least partially) in GPRs. + if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) { + assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args."); + // Store value to its stack slot. + SDValue PtrOff = + DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType()); + PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); + SDValue Store = + DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); + MemOpChains.push_back(Store); + const unsigned OriginalValNo = VA.getValNo(); + // Then load the GPRs from the stack + unsigned LoadOffset = 0; + auto HandleCustomVecRegLoc = [&]() { + assert(I != E && "Unexpected end of CCvalAssigns."); + assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && + "Expected custom RegLoc."); + CCValAssign RegVA = ArgLocs[I++]; + assert(RegVA.getValNo() == OriginalValNo && + "Custom MemLoc ValNo and custom RegLoc ValNo must match."); + SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, + DAG.getConstant(LoadOffset, dl, PtrVT)); + SDValue Load = DAG.getLoad(PtrVT, dl, Store, Add, MachinePointerInfo()); + MemOpChains.push_back(Load.getValue(1)); + RegsToPass.push_back(std::make_pair(RegVA.getLocReg(), Load)); + LoadOffset += PtrByteSize; + }; + + // In 64-bit there will be exactly 2 custom RegLocs that follow, and in + // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and + // R10. + HandleCustomVecRegLoc(); + HandleCustomVecRegLoc(); + + if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() && + ArgLocs[I].getValNo() == OriginalValNo) { + assert(!IsPPC64 && + "Only 2 custom RegLocs expected for 64-bit codegen."); + HandleCustomVecRegLoc(); + HandleCustomVecRegLoc(); + } + + continue; + } + if (VA.isMemLoc()) { SDValue PtrOff = DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType()); @@ -6935,11 +7391,15 @@ SDValue PPCTargetLowering::LowerCall_AIX( continue; } + if (!ValVT.isFloatingPoint()) + report_fatal_error( + "Unexpected register handling for calling convention."); + // Custom handling is used for GPR initializations for vararg float // arguments. assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && - ValVT.isFloatingPoint() && LocVT.isInteger() && - "Unexpected register handling for calling convention."); + LocVT.isInteger() && + "Custom register handling only expected for VarArg."); SDValue ArgAsInt = DAG.getBitcast(MVT::getIntegerVT(ValVT.getSizeInBits()), Arg); @@ -7370,18 +7830,29 @@ SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when /// possible. SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { - // Not FP, or using SPE? Not a fsel. - if (!Op.getOperand(0).getValueType().isFloatingPoint() || - !Op.getOperand(2).getValueType().isFloatingPoint() || Subtarget.hasSPE()) - return Op; - ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); - EVT ResVT = Op.getValueType(); EVT CmpVT = Op.getOperand(0).getValueType(); SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); SDLoc dl(Op); + + // Without power9-vector, we don't have native instruction for f128 comparison. + // Following transformation to libcall is needed for setcc: + // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE + if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) { + SDValue Z = DAG.getSetCC( + dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT), + LHS, RHS, CC); + SDValue Zero = DAG.getConstant(0, dl, Z.getValueType()); + return DAG.getSelectCC(dl, Z, Zero, TV, FV, ISD::SETNE); + } + + // Not FP, or using SPE? Not a fsel. + if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() || + Subtarget.hasSPE()) + return Op; + SDNodeFlags Flags = Op.getNode()->getFlags(); // We have xsmaxcdp/xsmincdp which are OK to emit even in the @@ -8583,6 +9054,18 @@ bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) { return Success; } +// Nondestructive check for convertTonNonDenormSingle. +bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) { + // Only convert if it loses info, since XXSPLTIDP should + // handle the other case. + APFloat APFloatToConvert = ArgAPFloat; + bool LosesInfo = true; + APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, + &LosesInfo); + + return (!LosesInfo && !APFloatToConvert.isDenormal()); +} + // If this is a case we can't handle, return null and let the default // expansion code take care of it. If we CAN select this case, and if it // selects to a single instruction, return Op. Otherwise, if we can codegen @@ -8604,16 +9087,20 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, // If it is a splat of a double, check if we can shrink it to a 32 bit // non-denormal float which when converted back to double gives us the same - // double. This is to exploit the XXSPLTIDP instruction.+ // If we lose precision, we use XXSPLTI32DX. + // double. This is to exploit the XXSPLTIDP instruction. + // If we lose precision, we use XXSPLTI32DX. if (BVNIsConstantSplat && (SplatBitSize == 64) && Subtarget.hasPrefixInstrs()) { - if (convertToNonDenormSingle(APSplatBits) && - (Op->getValueType(0) == MVT::v2f64)) { + // Check the type first to short-circuit so we don't modify APSplatBits if + // this block isn't executed. + if ((Op->getValueType(0) == MVT::v2f64) && + convertToNonDenormSingle(APSplatBits)) { SDValue SplatNode = DAG.getNode( PPCISD::XXSPLTI_SP_TO_DP, dl, MVT::v2f64, DAG.getTargetConstant(APSplatBits.getZExtValue(), dl, MVT::i32)); return DAG.getBitcast(Op.getValueType(), SplatNode); - } else { // We may lose precision, so we have to use XXSPLTI32DX. + } else { + // We may lose precision, so we have to use XXSPLTI32DX. uint32_t Hi = (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32); @@ -9279,7 +9766,8 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, // which is strictly wider than the loaded value by 8 bytes. So we need to // adjust the splat index to point to the correct address in memory. if (IsPermutedLoad) { - assert(isLittleEndian && "Unexpected permuted load on big endian target"); + assert((isLittleEndian || IsFourByte) && + "Unexpected size for permuted load on big endian target"); SplatIdx += IsFourByte ? 2 : 1; assert((SplatIdx < (IsFourByte ? 4 : 2)) && "Splat of a value outside of the loaded memory"); @@ -9294,6 +9782,11 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, else Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8; + // If the width of the load is the same as the width of the splat, + // loading with an offset would load the wrong memory. + if (LD->getValueType(0).getSizeInBits() == (IsFourByte ? 32 : 64)) + Offset = 0; + SDValue BasePtr = LD->getBasePtr(); if (Offset != 0) BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), @@ -9587,7 +10080,7 @@ static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, isDot = true; break; case Intrinsic::ppc_altivec_vcmpequd_p: - if (Subtarget.hasP8Altivec()) { + if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { CompareOpc = 199; isDot = true; } else @@ -9647,7 +10140,7 @@ static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsd_p: - if (Subtarget.hasP8Altivec()) { + if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { CompareOpc = 967; isDot = true; } else @@ -9666,7 +10159,7 @@ static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtud_p: - if (Subtarget.hasP8Altivec()) { + if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) { CompareOpc = 711; isDot = true; } else @@ -9870,7 +10363,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec, DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo : VecNo, - dl, MVT::i64)); + dl, getPointerTy(DAG.getDataLayout()))); RetOps.push_back(Extract); } return DAG.getMergeValues(RetOps, dl); @@ -9965,6 +10458,8 @@ SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, // Lower scalar BSWAP64 to xxbrd. SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); + if (!Subtarget.isPPC64()) + return Op; // MTVSRDD Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), Op.getOperand(0)); @@ -10037,14 +10532,39 @@ SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, "Should only be called for ISD::INSERT_VECTOR_ELT"); ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(2)); - // We have legal lowering for constant indices but not for variable ones. - if (!C) - return SDValue(); EVT VT = Op.getValueType(); SDLoc dl(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); + SDValue V3 = Op.getOperand(2); + + if (VT == MVT::v2f64 && C) + return Op; + + if (Subtarget.isISA3_1()) { + if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64()) + return SDValue(); + // On P10, we have legal lowering for constant and variable indices for + // integer vectors. + if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || + VT == MVT::v2i64) + return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, V2, V3); + // For f32 and f64 vectors, we have legal lowering for variable indices. + // For f32 we also have legal lowering when the element is loaded from + // memory. + if (VT == MVT::v4f32 || VT == MVT::v2f64) { + if (!C || (VT == MVT::v4f32 && dyn_cast<LoadSDNode>(V2))) + return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, V2, V3); + return Op; + } + } + + // Before P10, we have legal lowering for constant indices but not for + // variable ones. + if (!C) + return SDValue(); + // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. if (VT == MVT::v8i16 || VT == MVT::v16i8) { SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); @@ -10134,7 +10654,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, for (unsigned Idx = 0; Idx < NumVecs; ++Idx) { unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx; SDValue Elt = DAG.getNode(PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, Value, - DAG.getConstant(VecNum, dl, MVT::i64)); + DAG.getConstant(VecNum, dl, getPointerTy(DAG.getDataLayout()))); SDValue Store = DAG.getStore(StoreChain, dl, Elt, BasePtr, SN->getPointerInfo().getWithOffset(Idx * 16), @@ -10308,10 +10828,14 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); case ISD::JumpTable: return LowerJumpTable(Op, DAG); + case ISD::STRICT_FSETCC: + case ISD::STRICT_FSETCCS: case ISD::SETCC: return LowerSETCC(Op, DAG); case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); + case ISD::INLINEASM: + case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG); // Variable argument lowering. case ISD::VASTART: return LowerVASTART(Op, DAG); case ISD::VAARG: return LowerVAARG(Op, DAG); @@ -10464,7 +10988,7 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N, // Other Lowering Code //===----------------------------------------------------------------------===// -static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { +static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); Function *Func = Intrinsic::getDeclaration(M, Id); return Builder.CreateCall(Func, {}); @@ -10472,7 +10996,7 @@ static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { // The mappings for emitLeading/TrailingFence is taken from // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html -Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, +Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Ord == AtomicOrdering::SequentiallyConsistent) @@ -10482,7 +11006,7 @@ Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, return nullptr; } -Instruction *PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, +Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { @@ -10745,6 +11269,7 @@ MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); + Register SrwDestReg = RegInfo.createVirtualRegister(GPRC); Register Ptr1Reg; Register TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); @@ -10772,7 +11297,8 @@ MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( // stwcx. tmp4, ptr // bne- loopMBB // fallthrough --> exitMBB - // srw dest, tmpDest, shift + // srw SrwDest, tmpDest, shift + // rlwinm SrwDest, SrwDest, 0, 24 [16], 31 if (ptrA != ZeroReg) { Ptr1Reg = RegInfo.createVirtualRegister(RC); BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) @@ -10874,7 +11400,14 @@ MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( // exitMBB: // ... BB = exitMBB; - BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) + // Since the shift amount is not a constant, we need to clear + // the upper bits with a separate RLWINM. + BuildMI(*BB, BB->begin(), dl, TII->get(PPC::RLWINM), dest) + .addReg(SrwDestReg) + .addImm(0) + .addImm(is8bit ? 24 : 16) + .addImm(31); + BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), SrwDestReg) .addReg(TmpDestReg) .addReg(ShiftReg); return BB; @@ -12103,6 +12636,17 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } else if (MI.getOpcode() == PPC::PROBED_ALLOCA_32 || MI.getOpcode() == PPC::PROBED_ALLOCA_64) { return emitProbedAlloca(MI, BB); + } else if (MI.getOpcode() == PPC::SPLIT_QUADWORD) { + DebugLoc DL = MI.getDebugLoc(); + Register Src = MI.getOperand(2).getReg(); + Register Lo = MI.getOperand(0).getReg(); + Register Hi = MI.getOperand(1).getReg(); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY)) + .addDef(Lo) + .addUse(Src, 0, PPC::sub_gp8_x1); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY)) + .addDef(Hi) + .addUse(Src, 0, PPC::sub_gp8_x0); } else { llvm_unreachable("Unexpected instr type to insert"); } @@ -13434,7 +13978,7 @@ static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) { if (Operand.getOpcode() != ISD::LOAD) return SDValue(); - LoadSDNode *LD = dyn_cast<LoadSDNode>(Operand); + auto *LD = cast<LoadSDNode>(Operand); EVT MemoryType = LD->getMemoryVT(); // This transformation is only valid if the we are loading either a byte, @@ -13811,6 +14355,9 @@ SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, (Op1VT == MVT::i32 || Op1VT == MVT::i64 || (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); + if (ResVT == MVT::f128 && !Subtarget.hasP9Vector()) + return SDValue(); + if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() || cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) return SDValue(); @@ -13889,13 +14436,24 @@ static SDValue isScalarToVec(SDValue Op) { return SDValue(); } +// Fix up the shuffle mask to account for the fact that the result of +// scalar_to_vector is not in lane zero. This just takes all values in +// the ranges specified by the min/max indices and adds the number of +// elements required to ensure each element comes from the respective +// position in the valid lane. +// On little endian, that's just the corresponding element in the other +// half of the vector. On big endian, it is in the same half but right +// justified rather than left justified in that half. static void fixupShuffleMaskForPermutedSToV(SmallVectorImpl<int> &ShuffV, int LHSMaxIdx, int RHSMinIdx, - int RHSMaxIdx, int HalfVec) { + int RHSMaxIdx, int HalfVec, + unsigned ValidLaneWidth, + const PPCSubtarget &Subtarget) { for (int i = 0, e = ShuffV.size(); i < e; i++) { int Idx = ShuffV[i]; if ((Idx >= 0 && Idx < LHSMaxIdx) || (Idx >= RHSMinIdx && Idx < RHSMaxIdx)) - ShuffV[i] += HalfVec; + ShuffV[i] += + Subtarget.isLittleEndian() ? HalfVec : HalfVec - ValidLaneWidth; } } @@ -13904,7 +14462,8 @@ static void fixupShuffleMaskForPermutedSToV(SmallVectorImpl<int> &ShuffV, // (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C)))) // In such a case, just change the shuffle mask to extract the element // from the permuted index. -static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG) { +static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG, + const PPCSubtarget &Subtarget) { SDLoc dl(OrigSToV); EVT VT = OrigSToV.getValueType(); assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR && @@ -13918,8 +14477,14 @@ static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG) { // Can't handle non-const element indices or different vector types // for the input to the extract and the output of the scalar_to_vector. if (Idx && VT == OrigVector.getValueType()) { - SmallVector<int, 16> NewMask(VT.getVectorNumElements(), -1); - NewMask[VT.getVectorNumElements() / 2] = Idx->getZExtValue(); + unsigned NumElts = VT.getVectorNumElements(); + assert( + NumElts > 1 && + "Cannot produce a permuted scalar_to_vector for one element vector"); + SmallVector<int, 16> NewMask(NumElts, -1); + unsigned ResultInElt = NumElts / 2; + ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1; + NewMask[ResultInElt] = Idx->getZExtValue(); return DAG.getVectorShuffle(VT, dl, OrigVector, OrigVector, NewMask); } } @@ -13935,6 +14500,10 @@ static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG) { // Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute // to put the value into element zero. Adjust the shuffle mask so that the // vector can remain in permuted form (to prevent a swap prior to a shuffle). +// On big endian targets, this is still useful for SCALAR_TO_VECTOR +// nodes with elements smaller than doubleword because all the ways +// of getting scalar data into a vector register put the value in the +// rightmost element of the left half of the vector. SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG) const { SDValue LHS = SVN->getOperand(0); @@ -13943,10 +14512,14 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, int NumElts = LHS.getValueType().getVectorNumElements(); SDValue Res(SVN, 0); SDLoc dl(SVN); + bool IsLittleEndian = Subtarget.isLittleEndian(); - // None of these combines are useful on big endian systems since the ISA - // already has a big endian bias. - if (!Subtarget.isLittleEndian() || !Subtarget.hasVSX()) + // On big endian targets this is only useful for subtargets with direct moves. + // On little endian targets it would be useful for all subtargets with VSX. + // However adding special handling for LE subtargets without direct moves + // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8) + // which includes direct moves. + if (!Subtarget.hasDirectMove()) return Res; // If this is not a shuffle of a shuffle and the first element comes from @@ -13969,6 +14542,15 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, int NumEltsIn = SToVLHS ? SToVLHS.getValueType().getVectorNumElements() : SToVRHS.getValueType().getVectorNumElements(); int NumEltsOut = ShuffV.size(); + // The width of the "valid lane" (i.e. the lane that contains the value that + // is vectorized) needs to be expressed in terms of the number of elements + // of the shuffle. It is thereby the ratio of the values before and after + // any bitcast. + unsigned ValidLaneWidth = + SToVLHS ? SToVLHS.getValueType().getScalarSizeInBits() / + LHS.getValueType().getScalarSizeInBits() + : SToVRHS.getValueType().getScalarSizeInBits() / + RHS.getValueType().getScalarSizeInBits(); // Initially assume that neither input is permuted. These will be adjusted // accordingly if either input is. @@ -13979,18 +14561,26 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, // Get the permuted scalar to vector nodes for the source(s) that come from // ISD::SCALAR_TO_VECTOR. + // On big endian systems, this only makes sense for element sizes smaller + // than 64 bits since for 64-bit elements, all instructions already put + // the value into element zero. Since scalar size of LHS and RHS may differ + // after isScalarToVec, this should be checked using their own sizes. if (SToVLHS) { + if (!IsLittleEndian && SToVLHS.getValueType().getScalarSizeInBits() >= 64) + return Res; // Set up the values for the shuffle vector fixup. LHSMaxIdx = NumEltsOut / NumEltsIn; - SToVLHS = getSToVPermuted(SToVLHS, DAG); + SToVLHS = getSToVPermuted(SToVLHS, DAG, Subtarget); if (SToVLHS.getValueType() != LHS.getValueType()) SToVLHS = DAG.getBitcast(LHS.getValueType(), SToVLHS); LHS = SToVLHS; } if (SToVRHS) { + if (!IsLittleEndian && SToVRHS.getValueType().getScalarSizeInBits() >= 64) + return Res; RHSMinIdx = NumEltsOut; RHSMaxIdx = NumEltsOut / NumEltsIn + RHSMinIdx; - SToVRHS = getSToVPermuted(SToVRHS, DAG); + SToVRHS = getSToVPermuted(SToVRHS, DAG, Subtarget); if (SToVRHS.getValueType() != RHS.getValueType()) SToVRHS = DAG.getBitcast(RHS.getValueType(), SToVRHS); RHS = SToVRHS; @@ -14000,10 +14590,9 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, // The minimum and maximum indices that correspond to element zero for both // the LHS and RHS are computed and will control which shuffle mask entries // are to be changed. For example, if the RHS is permuted, any shuffle mask - // entries in the range [RHSMinIdx,RHSMaxIdx) will be incremented by - // HalfVec to refer to the corresponding element in the permuted vector. + // entries in the range [RHSMinIdx,RHSMaxIdx) will be adjusted. fixupShuffleMaskForPermutedSToV(ShuffV, LHSMaxIdx, RHSMinIdx, RHSMaxIdx, - HalfVec); + HalfVec, ValidLaneWidth, Subtarget); Res = DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV); // We may have simplified away the shuffle. We won't be able to do anything @@ -14013,12 +14602,13 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, Mask = cast<ShuffleVectorSDNode>(Res)->getMask(); } + SDValue TheSplat = IsLittleEndian ? RHS : LHS; // The common case after we commuted the shuffle is that the RHS is a splat // and we have elements coming in from the splat at indices that are not // conducive to using a merge. // Example: // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero> - if (!isSplatBV(RHS)) + if (!isSplatBV(TheSplat)) return Res; // We are looking for a mask such that all even elements are from @@ -14028,24 +14618,41 @@ SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN, // Adjust the mask so we are pulling in the same index from the splat // as the index from the interesting vector in consecutive elements. - // Example (even elements from first vector): - // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero> - if (Mask[0] < NumElts) - for (int i = 1, e = Mask.size(); i < e; i += 2) - ShuffV[i] = (ShuffV[i - 1] + NumElts); - // Example (odd elements from first vector): - // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero> - else - for (int i = 0, e = Mask.size(); i < e; i += 2) - ShuffV[i] = (ShuffV[i + 1] + NumElts); + if (IsLittleEndian) { + // Example (even elements from first vector): + // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero> + if (Mask[0] < NumElts) + for (int i = 1, e = Mask.size(); i < e; i += 2) + ShuffV[i] = (ShuffV[i - 1] + NumElts); + // Example (odd elements from first vector): + // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero> + else + for (int i = 0, e = Mask.size(); i < e; i += 2) + ShuffV[i] = (ShuffV[i + 1] + NumElts); + } else { + // Example (even elements from first vector): + // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1 + if (Mask[0] < NumElts) + for (int i = 0, e = Mask.size(); i < e; i += 2) + ShuffV[i] = ShuffV[i + 1] - NumElts; + // Example (odd elements from first vector): + // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1 + else + for (int i = 1, e = Mask.size(); i < e; i += 2) + ShuffV[i] = ShuffV[i - 1] - NumElts; + } // If the RHS has undefs, we need to remove them since we may have created // a shuffle that adds those instead of the splat value. - SDValue SplatVal = cast<BuildVectorSDNode>(RHS.getNode())->getSplatValue(); - RHS = DAG.getSplatBuildVector(RHS.getValueType(), dl, SplatVal); + SDValue SplatVal = + cast<BuildVectorSDNode>(TheSplat.getNode())->getSplatValue(); + TheSplat = DAG.getSplatBuildVector(TheSplat.getValueType(), dl, SplatVal); - Res = DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV); - return Res; + if (IsLittleEndian) + RHS = TheSplat; + else + LHS = TheSplat; + return DAG.getVectorShuffle(SVN->getValueType(0), dl, LHS, RHS, ShuffV); } SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, @@ -14084,7 +14691,15 @@ SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, return SDValue(); if (LSBase->getOpcode() == ISD::LOAD) { - SDLoc dl(SVN); + // If the load return value 0 has more than one user except the + // shufflevector instruction, it is not profitable to replace the + // shufflevector with a reverse load. + for (SDNode::use_iterator UI = LSBase->use_begin(), UE = LSBase->use_end(); + UI != UE; ++UI) + if (UI.getUse().getResNo() == 0 && UI->getOpcode() != ISD::VECTOR_SHUFFLE) + return SDValue(); + + SDLoc dl(LSBase); SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()}; return DAG.getMemIntrinsicNode( PPCISD::LOAD_VEC_BE, dl, DAG.getVTList(VT, MVT::Other), LoadOps, @@ -14092,6 +14707,12 @@ SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, } if (LSBase->getOpcode() == ISD::STORE) { + // If there are other uses of the shuffle, the swap cannot be avoided. + // Forcing the use of an X-Form (since swapped stores only have + // X-Forms) without removing the swap is unprofitable. + if (!SVN->hasOneUse()) + return SDValue(); + SDLoc dl(LSBase); SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0), LSBase->getBasePtr()}; @@ -14606,13 +15227,17 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, } } break; - case ISD::BSWAP: + case ISD::BSWAP: { // Turn BSWAP (LOAD) -> lhbrx/lwbrx. - if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && - N->getOperand(0).hasOneUse() && + // For subtargets without LDBRX, we can still do better than the default + // expansion even for 64-bit BSWAP (LOAD). + bool Is64BitBswapOn64BitTgt = + Subtarget.isPPC64() && N->getValueType(0) == MVT::i64; + bool IsSingleUseNormalLd = ISD::isNormalLoad(N->getOperand(0).getNode()) && + N->getOperand(0).hasOneUse(); + if (IsSingleUseNormalLd && (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || - (Subtarget.hasLDBRX() && Subtarget.isPPC64() && - N->getValueType(0) == MVT::i64))) { + (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) { SDValue Load = N->getOperand(0); LoadSDNode *LD = cast<LoadSDNode>(Load); // Create the byte-swapping load. @@ -14643,7 +15268,37 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, // Return N so it doesn't get rechecked! return SDValue(N, 0); } - break; + // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only + // before legalization so that the BUILD_PAIR is handled correctly. + if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt || + !IsSingleUseNormalLd) + return SDValue(); + LoadSDNode *LD = cast<LoadSDNode>(N->getOperand(0)); + + // Can't split volatile or atomic loads. + if (!LD->isSimple()) + return SDValue(); + SDValue BasePtr = LD->getBasePtr(); + SDValue Lo = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr, + LD->getPointerInfo(), LD->getAlignment()); + Lo = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Lo); + BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, + DAG.getIntPtrConstant(4, dl)); + MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand( + LD->getMemOperand(), 4, 4); + SDValue Hi = DAG.getLoad(MVT::i32, dl, LD->getChain(), BasePtr, NewMMO); + Hi = DAG.getNode(ISD::BSWAP, dl, MVT::i32, Hi); + SDValue Res; + if (Subtarget.isLittleEndian()) + Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Hi, Lo); + else + Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); + SDValue TF = + DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + Hi.getOperand(0).getValue(1), Lo.getOperand(0).getValue(1)); + DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), TF); + return Res; + } case PPCISD::VCMP: // If a VCMP_rec node already exists with exactly the same operands as this // node, use its result instead of this node (VCMP_rec computes both a CR6 @@ -15080,25 +15735,57 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, } else if ((Constraint == "wa" || Constraint == "wd" || Constraint == "wf" || Constraint == "wi") && Subtarget.hasVSX()) { - return std::make_pair(0U, &PPC::VSRCRegClass); + // A VSX register for either a scalar (FP) or vector. There is no + // support for single precision scalars on subtargets prior to Power8. + if (VT.isVector()) + return std::make_pair(0U, &PPC::VSRCRegClass); + if (VT == MVT::f32 && Subtarget.hasP8Vector()) + return std::make_pair(0U, &PPC::VSSRCRegClass); + return std::make_pair(0U, &PPC::VSFRCRegClass); } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { if (VT == MVT::f32 && Subtarget.hasP8Vector()) return std::make_pair(0U, &PPC::VSSRCRegClass); else return std::make_pair(0U, &PPC::VSFRCRegClass); + } else if (Constraint == "lr") { + if (VT == MVT::i64) + return std::make_pair(0U, &PPC::LR8RCRegClass); + else + return std::make_pair(0U, &PPC::LRRCRegClass); } - // If we name a VSX register, we can't defer to the base class because it - // will not recognize the correct register (their names will be VSL{0-31} - // and V{0-31} so they won't match). So we match them here. - if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { - int VSNum = atoi(Constraint.data() + 3); - assert(VSNum >= 0 && VSNum <= 63 && - "Attempted to access a vsr out of range"); - if (VSNum < 32) - return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); - return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); + // Handle special cases of physical registers that are not properly handled + // by the base class. + if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') { + // If we name a VSX register, we can't defer to the base class because it + // will not recognize the correct register (their names will be VSL{0-31} + // and V{0-31} so they won't match). So we match them here. + if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { + int VSNum = atoi(Constraint.data() + 3); + assert(VSNum >= 0 && VSNum <= 63 && + "Attempted to access a vsr out of range"); + if (VSNum < 32) + return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); + return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); + } + + // For float registers, we can't defer to the base class as it will match + // the SPILLTOVSRRC class. + if (Constraint.size() > 3 && Constraint[1] == 'f') { + int RegNum = atoi(Constraint.data() + 2); + if (RegNum > 31 || RegNum < 0) + report_fatal_error("Invalid floating point register number"); + if (VT == MVT::f32 || VT == MVT::i32) + return Subtarget.hasSPE() + ? std::make_pair(PPC::R0 + RegNum, &PPC::GPRCRegClass) + : std::make_pair(PPC::F0 + RegNum, &PPC::F4RCRegClass); + if (VT == MVT::f64 || VT == MVT::i64) + return Subtarget.hasSPE() + ? std::make_pair(PPC::S0 + RegNum, &PPC::SPERCRegClass) + : std::make_pair(PPC::F0 + RegNum, &PPC::F8RCRegClass); + } } + std::pair<unsigned, const TargetRegisterClass *> R = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); @@ -15115,10 +15802,19 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, &PPC::G8RCRegClass); // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. - if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { + if (!R.second && StringRef("{cc}").equals_insensitive(Constraint)) { R.first = PPC::CR0; R.second = &PPC::CRRCRegClass; } + // FIXME: This warning should ideally be emitted in the front end. + const auto &TM = getTargetMachine(); + if (Subtarget.isAIXABI() && !TM.getAIXExtendedAltivecABI()) { + if (((R.first >= PPC::V20 && R.first <= PPC::V31) || + (R.first >= PPC::VF20 && R.first <= PPC::VF31)) && + (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass)) + errs() << "warning: vector registers 20 to 32 are reserved in the " + "default AIX AltiVec ABI and cannot be used\n"; + } return R; } @@ -15365,6 +16061,22 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { + case Intrinsic::ppc_atomicrmw_xchg_i128: + case Intrinsic::ppc_atomicrmw_add_i128: + case Intrinsic::ppc_atomicrmw_sub_i128: + case Intrinsic::ppc_atomicrmw_nand_i128: + case Intrinsic::ppc_atomicrmw_and_i128: + case Intrinsic::ppc_atomicrmw_or_i128: + case Intrinsic::ppc_atomicrmw_xor_i128: + case Intrinsic::ppc_cmpxchg_i128: + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.memVT = MVT::i128; + Info.ptrVal = I.getArgOperand(0); + Info.offset = 0; + Info.align = Align(16); + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore | + MachineMemOperand::MOVolatile; + return true; case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_altivec_lvebx: @@ -15535,9 +16247,7 @@ bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { return isInt<16>(Imm) || isUInt<16>(Imm); } -bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, - unsigned, - unsigned, +bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align, MachineMemOperand::Flags, bool *Fast) const { if (DisablePPCUnaligned) @@ -15807,12 +16517,24 @@ bool PPCTargetLowering::useLoadStackGuardNode() const { return true; } -// Override to disable global variable loading on Linux. +// Override to disable global variable loading on Linux and insert AIX canary +// word declaration. void PPCTargetLowering::insertSSPDeclarations(Module &M) const { + if (Subtarget.isAIXABI()) { + M.getOrInsertGlobal(AIXSSPCanaryWordName, + Type::getInt8PtrTy(M.getContext())); + return; + } if (!Subtarget.isTargetLinux()) return TargetLowering::insertSSPDeclarations(M); } +Value *PPCTargetLowering::getSDagStackGuard(const Module &M) const { + if (Subtarget.isAIXABI()) + return M.getGlobalVariable(AIXSSPCanaryWordName); + return TargetLowering::getSDagStackGuard(M); +} + bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const { if (!VT.isSimple() || !Subtarget.hasVSX()) @@ -15826,10 +16548,8 @@ bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, case MVT::f32: case MVT::f64: if (Subtarget.hasPrefixInstrs()) { - // With prefixed instructions, we can materialize anything that can be - // represented with a 32-bit immediate, not just positive zero. - APFloat APFloatOfImm = Imm; - return convertToNonDenormSingle(APFloatOfImm); + // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP. + return true; } LLVM_FALLTHROUGH; case MVT::ppcf128: @@ -15960,9 +16680,7 @@ static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); SDValue Cmp = RHS.getOperand(0); SDValue Z = Cmp.getOperand(0); - auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1)); - - assert(Constant && "Constant Should not be a null pointer."); + auto *Constant = cast<ConstantSDNode>(Cmp.getOperand(1)); int64_t NegConstant = 0 - Constant->getSExtValue(); switch(cast<CondCodeSDNode>(Cmp.getOperand(2))->get()) { @@ -16411,3 +17129,463 @@ SDValue PPCTargetLowering::combineVSelect(SDNode *N, return SDValue(); } + +/// getAddrModeForFlags - Based on the set of address flags, select the most +/// optimal instruction format to match by. +PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const { + // This is not a node we should be handling here. + if (Flags == PPC::MOF_None) + return PPC::AM_None; + // Unaligned D-Forms are tried first, followed by the aligned D-Forms. + for (auto FlagSet : AddrModesMap.at(PPC::AM_DForm)) + if ((Flags & FlagSet) == FlagSet) + return PPC::AM_DForm; + for (auto FlagSet : AddrModesMap.at(PPC::AM_DSForm)) + if ((Flags & FlagSet) == FlagSet) + return PPC::AM_DSForm; + for (auto FlagSet : AddrModesMap.at(PPC::AM_DQForm)) + if ((Flags & FlagSet) == FlagSet) + return PPC::AM_DQForm; + // If no other forms are selected, return an X-Form as it is the most + // general addressing mode. + return PPC::AM_XForm; +} + +/// Set alignment flags based on whether or not the Frame Index is aligned. +/// Utilized when computing flags for address computation when selecting +/// load and store instructions. +static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet, + SelectionDAG &DAG) { + bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR)); + FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(IsAdd ? N.getOperand(0) : N); + if (!FI) + return; + const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + unsigned FrameIndexAlign = MFI.getObjectAlign(FI->getIndex()).value(); + // If this is (add $FI, $S16Imm), the alignment flags are already set + // based on the immediate. We just need to clear the alignment flags + // if the FI alignment is weaker. + if ((FrameIndexAlign % 4) != 0) + FlagSet &= ~PPC::MOF_RPlusSImm16Mult4; + if ((FrameIndexAlign % 16) != 0) + FlagSet &= ~PPC::MOF_RPlusSImm16Mult16; + // If the address is a plain FrameIndex, set alignment flags based on + // FI alignment. + if (!IsAdd) { + if ((FrameIndexAlign % 4) == 0) + FlagSet |= PPC::MOF_RPlusSImm16Mult4; + if ((FrameIndexAlign % 16) == 0) + FlagSet |= PPC::MOF_RPlusSImm16Mult16; + } +} + +/// Given a node, compute flags that are used for address computation when +/// selecting load and store instructions. The flags computed are stored in +/// FlagSet. This function takes into account whether the node is a constant, +/// an ADD, OR, or a constant, and computes the address flags accordingly. +static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet, + SelectionDAG &DAG) { + // Set the alignment flags for the node depending on if the node is + // 4-byte or 16-byte aligned. + auto SetAlignFlagsForImm = [&](uint64_t Imm) { + if ((Imm & 0x3) == 0) + FlagSet |= PPC::MOF_RPlusSImm16Mult4; + if ((Imm & 0xf) == 0) + FlagSet |= PPC::MOF_RPlusSImm16Mult16; + }; + + if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { + // All 32-bit constants can be computed as LIS + Disp. + const APInt &ConstImm = CN->getAPIntValue(); + if (ConstImm.isSignedIntN(32)) { // Flag to handle 32-bit constants. + FlagSet |= PPC::MOF_AddrIsSImm32; + SetAlignFlagsForImm(ConstImm.getZExtValue()); + setAlignFlagsForFI(N, FlagSet, DAG); + } + if (ConstImm.isSignedIntN(34)) // Flag to handle 34-bit constants. + FlagSet |= PPC::MOF_RPlusSImm34; + else // Let constant materialization handle large constants. + FlagSet |= PPC::MOF_NotAddNorCst; + } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) { + // This address can be represented as an addition of: + // - Register + Imm16 (possibly a multiple of 4/16) + // - Register + Imm34 + // - Register + PPCISD::Lo + // - Register + Register + // In any case, we won't have to match this as Base + Zero. + SDValue RHS = N.getOperand(1); + if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(RHS)) { + const APInt &ConstImm = CN->getAPIntValue(); + if (ConstImm.isSignedIntN(16)) { + FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates. + SetAlignFlagsForImm(ConstImm.getZExtValue()); + setAlignFlagsForFI(N, FlagSet, DAG); + } + if (ConstImm.isSignedIntN(34)) + FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates. + else + FlagSet |= PPC::MOF_RPlusR; // Register. + } else if (RHS.getOpcode() == PPCISD::Lo && + !cast<ConstantSDNode>(RHS.getOperand(1))->getZExtValue()) + FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo. + else + FlagSet |= PPC::MOF_RPlusR; + } else { // The address computation is not a constant or an addition. + setAlignFlagsForFI(N, FlagSet, DAG); + FlagSet |= PPC::MOF_NotAddNorCst; + } +} + +/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute +/// the address flags of the load/store instruction that is to be matched. +unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N, + SelectionDAG &DAG) const { + unsigned FlagSet = PPC::MOF_None; + + // Compute subtarget flags. + if (!Subtarget.hasP9Vector()) + FlagSet |= PPC::MOF_SubtargetBeforeP9; + else { + FlagSet |= PPC::MOF_SubtargetP9; + if (Subtarget.hasPrefixInstrs()) + FlagSet |= PPC::MOF_SubtargetP10; + } + if (Subtarget.hasSPE()) + FlagSet |= PPC::MOF_SubtargetSPE; + + // Mark this as something we don't want to handle here if it is atomic + // or pre-increment instruction. + if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Parent)) + if (LSB->isIndexed()) + return PPC::MOF_None; + + // Compute in-memory type flags. This is based on if there are scalars, + // floats or vectors. + const MemSDNode *MN = dyn_cast<MemSDNode>(Parent); + assert(MN && "Parent should be a MemSDNode!"); + EVT MemVT = MN->getMemoryVT(); + unsigned Size = MemVT.getSizeInBits(); + if (MemVT.isScalarInteger()) { + assert(Size <= 64 && "Not expecting scalar integers larger than 8 bytes!"); + if (Size < 32) + FlagSet |= PPC::MOF_SubWordInt; + else if (Size == 32) + FlagSet |= PPC::MOF_WordInt; + else + FlagSet |= PPC::MOF_DoubleWordInt; + } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors. + if (Size == 128) + FlagSet |= PPC::MOF_Vector; + else if (Size == 256) + FlagSet |= PPC::MOF_Vector256; + else + llvm_unreachable("Not expecting illegal vectors!"); + } else { // Floating point type: can be scalar, f128 or vector types. + if (Size == 32 || Size == 64) + FlagSet |= PPC::MOF_ScalarFloat; + else if (MemVT == MVT::f128 || MemVT.isVector()) + FlagSet |= PPC::MOF_Vector; + else + llvm_unreachable("Not expecting illegal scalar floats!"); + } + + // Compute flags for address computation. + computeFlagsForAddressComputation(N, FlagSet, DAG); + + // Compute type extension flags. + if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Parent)) { + switch (LN->getExtensionType()) { + case ISD::SEXTLOAD: + FlagSet |= PPC::MOF_SExt; + break; + case ISD::EXTLOAD: + case ISD::ZEXTLOAD: + FlagSet |= PPC::MOF_ZExt; + break; + case ISD::NON_EXTLOAD: + FlagSet |= PPC::MOF_NoExt; + break; + } + } else + FlagSet |= PPC::MOF_NoExt; + + // For integers, no extension is the same as zero extension. + // We set the extension mode to zero extension so we don't have + // to add separate entries in AddrModesMap for loads and stores. + if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) { + FlagSet |= PPC::MOF_ZExt; + FlagSet &= ~PPC::MOF_NoExt; + } + + // If we don't have prefixed instructions, 34-bit constants should be + // treated as PPC::MOF_NotAddNorCst so they can match D-Forms. + bool IsNonP1034BitConst = + ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) & + FlagSet) == PPC::MOF_RPlusSImm34; + if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR && + IsNonP1034BitConst) + FlagSet |= PPC::MOF_NotAddNorCst; + + return FlagSet; +} + +/// SelectForceXFormMode - Given the specified address, force it to be +/// represented as an indexed [r+r] operation (an XForm instruction). +PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp, + SDValue &Base, + SelectionDAG &DAG) const { + + PPC::AddrMode Mode = PPC::AM_XForm; + int16_t ForceXFormImm = 0; + if (provablyDisjointOr(DAG, N) && + !isIntS16Immediate(N.getOperand(1), ForceXFormImm)) { + Disp = N.getOperand(0); + Base = N.getOperand(1); + return Mode; + } + + // If the address is the result of an add, we will utilize the fact that the + // address calculation includes an implicit add. However, we can reduce + // register pressure if we do not materialize a constant just for use as the + // index register. We only get rid of the add if it is not an add of a + // value and a 16-bit signed constant and both have a single use. + if (N.getOpcode() == ISD::ADD && + (!isIntS16Immediate(N.getOperand(1), ForceXFormImm) || + !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { + Disp = N.getOperand(0); + Base = N.getOperand(1); + return Mode; + } + + // Otherwise, use R0 as the base register. + Disp = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, + N.getValueType()); + Base = N; + + return Mode; +} + +// If we happen to match to an aligned D-Form, check if the Frame Index is +// adequately aligned. If it is not, reset the mode to match to X-Form. +static void setXFormForUnalignedFI(SDValue N, unsigned Flags, + PPC::AddrMode &Mode) { + if (!isa<FrameIndexSDNode>(N)) + return; + if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) || + (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16))) + Mode = PPC::AM_XForm; +} + +/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode), +/// compute the address flags of the node, get the optimal address mode based +/// on the flags, and set the Base and Disp based on the address mode. +PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent, + SDValue N, SDValue &Disp, + SDValue &Base, + SelectionDAG &DAG, + MaybeAlign Align) const { + SDLoc DL(Parent); + + // Compute the address flags. + unsigned Flags = computeMOFlags(Parent, N, DAG); + + // Get the optimal address mode based on the Flags. + PPC::AddrMode Mode = getAddrModeForFlags(Flags); + + // If the address mode is DS-Form or DQ-Form, check if the FI is aligned. + // Select an X-Form load if it is not. + setXFormForUnalignedFI(N, Flags, Mode); + + // Set Base and Disp accordingly depending on the address mode. + switch (Mode) { + case PPC::AM_DForm: + case PPC::AM_DSForm: + case PPC::AM_DQForm: { + // This is a register plus a 16-bit immediate. The base will be the + // register and the displacement will be the immediate unless it + // isn't sufficiently aligned. + if (Flags & PPC::MOF_RPlusSImm16) { + SDValue Op0 = N.getOperand(0); + SDValue Op1 = N.getOperand(1); + int16_t Imm = cast<ConstantSDNode>(Op1)->getAPIntValue().getZExtValue(); + if (!Align || isAligned(*Align, Imm)) { + Disp = DAG.getTargetConstant(Imm, DL, N.getValueType()); + Base = Op0; + if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0)) { + Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); + fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); + } + break; + } + } + // This is a register plus the @lo relocation. The base is the register + // and the displacement is the global address. + else if (Flags & PPC::MOF_RPlusLo) { + Disp = N.getOperand(1).getOperand(0); // The global address. + assert(Disp.getOpcode() == ISD::TargetGlobalAddress || + Disp.getOpcode() == ISD::TargetGlobalTLSAddress || + Disp.getOpcode() == ISD::TargetConstantPool || + Disp.getOpcode() == ISD::TargetJumpTable); + Base = N.getOperand(0); + break; + } + // This is a constant address at most 32 bits. The base will be + // zero or load-immediate-shifted and the displacement will be + // the low 16 bits of the address. + else if (Flags & PPC::MOF_AddrIsSImm32) { + auto *CN = cast<ConstantSDNode>(N); + EVT CNType = CN->getValueType(0); + uint64_t CNImm = CN->getZExtValue(); + // If this address fits entirely in a 16-bit sext immediate field, codegen + // this as "d, 0". + int16_t Imm; + if (isIntS16Immediate(CN, Imm) && (!Align || isAligned(*Align, Imm))) { + Disp = DAG.getTargetConstant(Imm, DL, CNType); + Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, + CNType); + break; + } + // Handle 32-bit sext immediate with LIS + Addr mode. + if ((CNType == MVT::i32 || isInt<32>(CNImm)) && + (!Align || isAligned(*Align, CNImm))) { + int32_t Addr = (int32_t)CNImm; + // Otherwise, break this down into LIS + Disp. + Disp = DAG.getTargetConstant((int16_t)Addr, DL, MVT::i32); + Base = + DAG.getTargetConstant((Addr - (int16_t)Addr) >> 16, DL, MVT::i32); + uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8; + Base = SDValue(DAG.getMachineNode(LIS, DL, CNType, Base), 0); + break; + } + } + // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable. + Disp = DAG.getTargetConstant(0, DL, getPointerTy(DAG.getDataLayout())); + if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { + Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); + fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); + } else + Base = N; + break; + } + case PPC::AM_None: + break; + default: { // By default, X-Form is always available to be selected. + // When a frame index is not aligned, we also match by XForm. + FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N); + Base = FI ? N : N.getOperand(1); + Disp = FI ? DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, + N.getValueType()) + : N.getOperand(0); + break; + } + } + return Mode; +} + +CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC, + bool Return, + bool IsVarArg) const { + switch (CC) { + case CallingConv::Cold: + return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF_FIS); + default: + return CC_PPC64_ELF_FIS; + } +} + +TargetLowering::AtomicExpansionKind +PPCTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); + if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128) + return AtomicExpansionKind::MaskedIntrinsic; + return TargetLowering::shouldExpandAtomicRMWInIR(AI); +} + +TargetLowering::AtomicExpansionKind +PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + unsigned Size = AI->getPointerOperand() + ->getType() + ->getPointerElementType() + ->getPrimitiveSizeInBits(); + if (EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && Size == 128) + return AtomicExpansionKind::MaskedIntrinsic; + return TargetLowering::shouldExpandAtomicCmpXchgInIR(AI); +} + +static Intrinsic::ID +getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) { + switch (BinOp) { + default: + llvm_unreachable("Unexpected AtomicRMW BinOp"); + case AtomicRMWInst::Xchg: + return Intrinsic::ppc_atomicrmw_xchg_i128; + case AtomicRMWInst::Add: + return Intrinsic::ppc_atomicrmw_add_i128; + case AtomicRMWInst::Sub: + return Intrinsic::ppc_atomicrmw_sub_i128; + case AtomicRMWInst::And: + return Intrinsic::ppc_atomicrmw_and_i128; + case AtomicRMWInst::Or: + return Intrinsic::ppc_atomicrmw_or_i128; + case AtomicRMWInst::Xor: + return Intrinsic::ppc_atomicrmw_xor_i128; + case AtomicRMWInst::Nand: + return Intrinsic::ppc_atomicrmw_nand_i128; + } +} + +Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic( + IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, + Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const { + assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && + "Only support quadword now"); + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + Type *ValTy = cast<PointerType>(AlignedAddr->getType())->getElementType(); + assert(ValTy->getPrimitiveSizeInBits() == 128); + Function *RMW = Intrinsic::getDeclaration( + M, getIntrinsicForAtomicRMWBinOp128(AI->getOperation())); + Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Value *IncrLo = Builder.CreateTrunc(Incr, Int64Ty, "incr_lo"); + Value *IncrHi = + Builder.CreateTrunc(Builder.CreateLShr(Incr, 64), Int64Ty, "incr_hi"); + Value *Addr = + Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext())); + Value *LoHi = Builder.CreateCall(RMW, {Addr, IncrLo, IncrHi}); + Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); + Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); + Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); + Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); + return Builder.CreateOr( + Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64"); +} + +Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( + IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, + Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { + assert(EnableQuadwordAtomics && Subtarget.hasQuadwordAtomics() && + "Only support quadword now"); + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + Type *ValTy = cast<PointerType>(AlignedAddr->getType())->getElementType(); + assert(ValTy->getPrimitiveSizeInBits() == 128); + Function *IntCmpXchg = + Intrinsic::getDeclaration(M, Intrinsic::ppc_cmpxchg_i128); + Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Value *CmpLo = Builder.CreateTrunc(CmpVal, Int64Ty, "cmp_lo"); + Value *CmpHi = + Builder.CreateTrunc(Builder.CreateLShr(CmpVal, 64), Int64Ty, "cmp_hi"); + Value *NewLo = Builder.CreateTrunc(NewVal, Int64Ty, "new_lo"); + Value *NewHi = + Builder.CreateTrunc(Builder.CreateLShr(NewVal, 64), Int64Ty, "new_hi"); + Value *Addr = + Builder.CreateBitCast(AlignedAddr, Type::getInt8PtrTy(M->getContext())); + emitLeadingFence(Builder, CI, Ord); + Value *LoHi = + Builder.CreateCall(IntCmpXchg, {Addr, CmpLo, CmpHi, NewLo, NewHi}); + emitTrailingFence(Builder, CI, Ord); + Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); + Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); + Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); + Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); + return Builder.CreateOr( + Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64"); +} |