diff options
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp')
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 814 |
1 files changed, 676 insertions, 138 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 5760132e44a0..1c1dae8f953f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -53,22 +53,24 @@ bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, const Function &F = DAG.getMachineFunction().getFunction(); // First, check if tail calls have been disabled in this function. - if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true") + if (F.getFnAttribute("disable-tail-calls").getValueAsBool()) return false; // Conservatively require the attributes of the call to match those of - // the return. Ignore NoAlias and NonNull because they don't affect the + // the return. Ignore following attributes because they don't affect the // call sequence. - AttributeList CallerAttrs = F.getAttributes(); - if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex) - .removeAttribute(Attribute::NoAlias) - .removeAttribute(Attribute::NonNull) - .hasAttributes()) + AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex); + for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable, + Attribute::DereferenceableOrNull, Attribute::NoAlias, + Attribute::NonNull}) + CallerAttrs.removeAttribute(Attr); + + if (CallerAttrs.hasAttributes()) return false; // It's not safe to eliminate the sign / zero extension of the return value. - if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) || - CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) + if (CallerAttrs.contains(Attribute::ZExt) || + CallerAttrs.contains(Attribute::SExt)) return false; // Check if the only use is a function return node. @@ -114,14 +116,21 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf); + IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync); IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); - Alignment = Call->getParamAlign(ArgIdx); - ByValType = nullptr; - if (IsByVal) - ByValType = Call->getParamByValType(ArgIdx); - PreallocatedType = nullptr; + Alignment = Call->getParamStackAlign(ArgIdx); + IndirectType = nullptr; + assert(IsByVal + IsPreallocated + IsInAlloca <= 1 && + "multiple ABI attributes?"); + if (IsByVal) { + IndirectType = Call->getParamByValType(ArgIdx); + if (!Alignment) + Alignment = Call->getParamAlign(ArgIdx); + } if (IsPreallocated) - PreallocatedType = Call->getParamPreallocatedType(ArgIdx); + IndirectType = Call->getParamPreallocatedType(ArgIdx); + if (IsInAlloca) + IndirectType = Call->getParamInAllocaType(ArgIdx); } /// Generate a libcall taking the given operands as arguments and returning a @@ -194,9 +203,8 @@ bool TargetLowering::findOptimalMemOpLowering( // equal to DstAlign (or zero). VT = MVT::i64; if (Op.isFixedDstAlign()) - while ( - Op.getDstAlign() < (VT.getSizeInBits() / 8) && - !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign().value())) + while (Op.getDstAlign() < (VT.getSizeInBits() / 8) && + !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign())) VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); assert(VT.isInteger()); @@ -250,7 +258,7 @@ bool TargetLowering::findOptimalMemOpLowering( bool Fast; if (NumMemOps && Op.allowOverlap() && NewVTSize < Size && allowsMisalignedMemoryAccesses( - VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign().value() : 1, + VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), MachineMemOperand::MONone, &Fast) && Fast) VTSize = Size; @@ -502,7 +510,7 @@ bool TargetLowering::ShrinkDemandedConstant(SDValue Op, case ISD::AND: case ISD::OR: { auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); - if (!Op1C) + if (!Op1C || Op1C->isOpaque()) return false; // If this is a 'not' op, don't touch it because that's a canonical form. @@ -971,11 +979,12 @@ bool TargetLowering::SimplifyDemandedBits( Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); return false; // Don't fall through, will infinitely loop. case ISD::LOAD: { - LoadSDNode *LD = cast<LoadSDNode>(Op); + auto *LD = cast<LoadSDNode>(Op); if (getTargetConstantFromLoad(LD)) { Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); return false; // Don't fall through, will infinitely loop. - } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { + } + if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { // If this is a ZEXTLoad and we are looking at the loaded value. EVT MemVT = LD->getMemoryVT(); unsigned MemBits = MemVT.getScalarSizeInBits(); @@ -2012,7 +2021,7 @@ bool TargetLowering::SimplifyDemandedBits( const APInt *ShAmtC = TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts); - if (!ShAmtC) + if (!ShAmtC || ShAmtC->uge(BitWidth)) break; uint64_t ShVal = ShAmtC->getZExtValue(); @@ -2267,10 +2276,8 @@ bool TargetLowering::SimplifyDemandedBits( if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) { // Avoid folding to a constant if any OpaqueConstant is involved. const SDNode *N = Op.getNode(); - for (SDNodeIterator I = SDNodeIterator::begin(N), - E = SDNodeIterator::end(N); - I != E; ++I) { - SDNode *Op = *I; + for (SDNode *Op : + llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) { if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) if (C->isOpaque()) return false; @@ -2417,6 +2424,27 @@ bool TargetLowering::SimplifyDemandedVectorElts( KnownUndef.setAllBits(); return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); } + SDValue ScalarSrc = Op.getOperand(0); + if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { + SDValue Src = ScalarSrc.getOperand(0); + SDValue Idx = ScalarSrc.getOperand(1); + EVT SrcVT = Src.getValueType(); + + ElementCount SrcEltCnt = SrcVT.getVectorElementCount(); + + if (SrcEltCnt.isScalable()) + return false; + + unsigned NumSrcElts = SrcEltCnt.getFixedValue(); + if (isNullConstant(Idx)) { + APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0); + APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts); + APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts); + if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero, + TLO, Depth + 1)) + return true; + } + } KnownUndef.setHighBits(NumElts - 1); break; } @@ -3028,6 +3056,19 @@ const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const { return nullptr; } +bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( + SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, + bool PoisonOnly, unsigned Depth) const { + assert( + (Op.getOpcode() >= ISD::BUILTIN_OP_END || + Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || + Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || + Op.getOpcode() == ISD::INTRINSIC_VOID) && + "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op" + " is a target node!"); + return false; +} + bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN, @@ -4502,40 +4543,39 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, GA->getValueType(0), Offset + GA->getOffset())); return; - } else if ((C = dyn_cast<ConstantSDNode>(Op)) && - ConstraintLetter != 's') { + } + if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') { // gcc prints these as sign extended. Sign extend value to 64 bits // now; without this it would get ZExt'd later in // ScheduleDAGSDNodes::EmitNode, which is very generic. bool IsBool = C->getConstantIntValue()->getBitWidth() == 1; BooleanContent BCont = getBooleanContents(MVT::i64); - ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont) - : ISD::SIGN_EXTEND; - int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() - : C->getSExtValue(); - Ops.push_back(DAG.getTargetConstant(Offset + ExtVal, - SDLoc(C), MVT::i64)); + ISD::NodeType ExtOpc = + IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND; + int64_t ExtVal = + ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue(); + Ops.push_back( + DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64)); return; - } else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && - ConstraintLetter != 'n') { + } + if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') { Ops.push_back(DAG.getTargetBlockAddress( BA->getBlockAddress(), BA->getValueType(0), Offset + BA->getOffset(), BA->getTargetFlags())); return; - } else { - const unsigned OpCode = Op.getOpcode(); - if (OpCode == ISD::ADD || OpCode == ISD::SUB) { - if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0)))) - Op = Op.getOperand(1); - // Subtraction is not commutative. - else if (OpCode == ISD::ADD && - (C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))) - Op = Op.getOperand(0); - else - return; - Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); - continue; - } + } + const unsigned OpCode = Op.getOpcode(); + if (OpCode == ISD::ADD || OpCode == ISD::SUB) { + if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0)))) + Op = Op.getOperand(1); + // Subtraction is not commutative. + else if (OpCode == ISD::ADD && + (C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))) + Op = Op.getOperand(0); + else + return; + Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); + continue; } return; } @@ -4565,11 +4605,10 @@ TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI, if (!isLegalRC(*RI, *RC)) continue; - for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); - I != E; ++I) { - if (RegName.equals_lower(RI->getRegAsmName(*I))) { + for (const MCPhysReg &PR : *RC) { + if (RegName.equals_insensitive(RI->getRegAsmName(PR))) { std::pair<unsigned, const TargetRegisterClass *> S = - std::make_pair(*I, RC); + std::make_pair(PR, RC); // If this register class has the requested value type, return it, // otherwise keep searching and return the first class found @@ -5033,16 +5072,17 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, return SDValue(); SDValue Shift, Factor; - if (VT.isFixedLengthVector()) { + if (Op1.getOpcode() == ISD::BUILD_VECTOR) { Shift = DAG.getBuildVector(ShVT, dl, Shifts); Factor = DAG.getBuildVector(VT, dl, Factors); - } else if (VT.isScalableVector()) { + } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) { assert(Shifts.size() == 1 && Factors.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable " "vectors"); Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]); Factor = DAG.getSplatVector(VT, dl, Factors[0]); } else { + assert(isa<ConstantSDNode>(Op1) && "Expected a constant"); Shift = Shifts[0]; Factor = Factors[0]; } @@ -5084,11 +5124,25 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); EVT ShSVT = ShVT.getScalarType(); unsigned EltBits = VT.getScalarSizeInBits(); + EVT MulVT; // Check to see if we can do this. // FIXME: We should be more aggressive here. - if (!isTypeLegal(VT)) - return SDValue(); + if (!isTypeLegal(VT)) { + // Limit this to simple scalars for now. + if (VT.isVector() || !VT.isSimple()) + return SDValue(); + + // If this type will be promoted to a large enough type with a legal + // multiply operation, we can go ahead and do this transform. + if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) + return SDValue(); + + MulVT = getTypeToTransformTo(*DAG.getContext(), VT); + if (MulVT.getSizeInBits() < (2 * EltBits) || + !isOperationLegal(ISD::MUL, MulVT)) + return SDValue(); + } // If the sdiv has an 'exact' bit we can use a simpler lowering. if (N->getFlags().hasExact()) @@ -5134,12 +5188,12 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, return SDValue(); SDValue MagicFactor, Factor, Shift, ShiftMask; - if (VT.isFixedLengthVector()) { + if (N1.getOpcode() == ISD::BUILD_VECTOR) { MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); Factor = DAG.getBuildVector(VT, dl, Factors); Shift = DAG.getBuildVector(ShVT, dl, Shifts); ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks); - } else if (VT.isScalableVector()) { + } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { assert(MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable " @@ -5149,6 +5203,7 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]); ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]); } else { + assert(isa<ConstantSDNode>(N1) && "Expected a constant"); MagicFactor = MagicFactors[0]; Factor = Factors[0]; Shift = Shifts[0]; @@ -5157,17 +5212,32 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, // Multiply the numerator (operand 0) by the magic value. // FIXME: We should support doing a MUL in a wider type. - SDValue Q; - if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT) - : isOperationLegalOrCustom(ISD::MULHS, VT)) - Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor); - else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT) - : isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) { - SDValue LoHi = - DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor); - Q = SDValue(LoHi.getNode(), 1); - } else - return SDValue(); // No mulhs or equivalent. + auto GetMULHS = [&](SDValue X, SDValue Y) { + // If the type isn't legal, use a wider mul of the the type calculated + // earlier. + if (!isTypeLegal(VT)) { + X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X); + Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y); + Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); + Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, + DAG.getShiftAmountConstant(EltBits, MulVT, dl)); + return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); + } + + if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) + return DAG.getNode(ISD::MULHS, dl, VT, X, Y); + if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { + SDValue LoHi = + DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); + return SDValue(LoHi.getNode(), 1); + } + return SDValue(); + }; + + SDValue Q = GetMULHS(N0, MagicFactor); + if (!Q) + return SDValue(); + Created.push_back(Q.getNode()); // (Optionally) Add/subtract the numerator using Factor. @@ -5202,11 +5272,25 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); EVT ShSVT = ShVT.getScalarType(); unsigned EltBits = VT.getScalarSizeInBits(); + EVT MulVT; // Check to see if we can do this. // FIXME: We should be more aggressive here. - if (!isTypeLegal(VT)) - return SDValue(); + if (!isTypeLegal(VT)) { + // Limit this to simple scalars for now. + if (VT.isVector() || !VT.isSimple()) + return SDValue(); + + // If this type will be promoted to a large enough type with a legal + // multiply operation, we can go ahead and do this transform. + if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) + return SDValue(); + + MulVT = getTypeToTransformTo(*DAG.getContext(), VT); + if (MulVT.getSizeInBits() < (2 * EltBits) || + !isOperationLegal(ISD::MUL, MulVT)) + return SDValue(); + } bool UseNPQ = false; SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors; @@ -5216,7 +5300,7 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, return false; // FIXME: We should use a narrower constant when the upper // bits are known to be zero. - APInt Divisor = C->getAPIntValue(); + const APInt& Divisor = C->getAPIntValue(); APInt::mu magics = Divisor.magicu(); unsigned PreShift = 0, PostShift = 0; @@ -5261,12 +5345,12 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, return SDValue(); SDValue PreShift, PostShift, MagicFactor, NPQFactor; - if (VT.isFixedLengthVector()) { + if (N1.getOpcode() == ISD::BUILD_VECTOR) { PreShift = DAG.getBuildVector(ShVT, dl, PreShifts); MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors); PostShift = DAG.getBuildVector(ShVT, dl, PostShifts); - } else if (VT.isScalableVector()) { + } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { assert(PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"); @@ -5275,6 +5359,7 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]); PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]); } else { + assert(isa<ConstantSDNode>(N1) && "Expected a constant"); PreShift = PreShifts[0]; MagicFactor = MagicFactors[0]; PostShift = PostShifts[0]; @@ -5286,11 +5371,20 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, // FIXME: We should support doing a MUL in a wider type. auto GetMULHU = [&](SDValue X, SDValue Y) { - if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT) - : isOperationLegalOrCustom(ISD::MULHU, VT)) + // If the type isn't legal, use a wider mul of the the type calculated + // earlier. + if (!isTypeLegal(VT)) { + X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X); + Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y); + Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); + Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, + DAG.getShiftAmountConstant(EltBits, MulVT, dl)); + return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); + } + + if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) return DAG.getNode(ISD::MULHU, dl, VT, X, Y); - if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT) - : isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) { + if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { SDValue LoHi = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); return SDValue(LoHi.getNode(), 1); @@ -5398,11 +5492,11 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, EVT VT = REMNode.getValueType(); EVT SVT = VT.getScalarType(); - EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); + EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); EVT ShSVT = ShVT.getScalarType(); // If MUL is unavailable, we cannot proceed in any case. - if (!isOperationLegalOrCustom(ISD::MUL, VT)) + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) return SDValue(); bool ComparingWithAllZeros = true; @@ -5511,7 +5605,7 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, return SDValue(); SDValue PVal, KVal, QVal; - if (VT.isVector()) { + if (D.getOpcode() == ISD::BUILD_VECTOR) { if (HadTautologicalLanes) { // Try to turn PAmts into a splat, since we don't care about the values // that are currently '0'. If we can't, just keep '0'`s. @@ -5525,6 +5619,13 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, PVal = DAG.getBuildVector(VT, DL, PAmts); KVal = DAG.getBuildVector(ShVT, DL, KAmts); QVal = DAG.getBuildVector(VT, DL, QAmts); + } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { + assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && + "Expected matchBinaryPredicate to return one element for " + "SPLAT_VECTORs"); + PVal = DAG.getSplatVector(VT, DL, PAmts[0]); + KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]); + QVal = DAG.getSplatVector(VT, DL, QAmts[0]); } else { PVal = PAmts[0]; KVal = KAmts[0]; @@ -5532,7 +5633,7 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, } if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) { - if (!isOperationLegalOrCustom(ISD::SUB, VT)) + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) return SDValue(); // FIXME: Could/should use `ISD::ADD`? assert(CompTargetNode.getValueType() == N.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."); @@ -5547,12 +5648,10 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, // divisors as a performance improvement, since rotating by 0 is a no-op. if (HadEvenDivisor) { // We need ROTR to do this. - if (!isOperationLegalOrCustom(ISD::ROTR, VT)) + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) return SDValue(); - SDNodeFlags Flags; - Flags.setExact(true); // UREM: (rotr (mul N, P), K) - Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags); + Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); Created.push_back(Op0.getNode()); } @@ -5577,6 +5676,8 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE); Created.push_back(TautologicalInvertedChannels.getNode()); + // NOTE: we avoid letting illegal types through even if we're before legalize + // ops – legalization has a hard time producing good code for this. if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) { // If we have a vector select, let's replace the comparison results in the // affected lanes with the correct tautological result. @@ -5587,6 +5688,8 @@ TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, } // Else, we can just invert the comparison result in the appropriate lanes. + // + // NOTE: see the note above VSELECT above. if (isOperationLegalOrCustom(ISD::XOR, SETCCVT)) return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC, TautologicalInvertedChannels); @@ -5638,11 +5741,12 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, EVT VT = REMNode.getValueType(); EVT SVT = VT.getScalarType(); - EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); + EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); EVT ShSVT = ShVT.getScalarType(); - // If MUL is unavailable, we cannot proceed in any case. - if (!isOperationLegalOrCustom(ISD::MUL, VT)) + // If we are after ops legalization, and MUL is unavailable, we can not + // proceed. + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) return SDValue(); // TODO: Could support comparing with non-zero too. @@ -5755,7 +5859,7 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, return SDValue(); SDValue PVal, AVal, KVal, QVal; - if (VT.isFixedLengthVector()) { + if (D.getOpcode() == ISD::BUILD_VECTOR) { if (HadOneDivisor) { // Try to turn PAmts into a splat, since we don't care about the values // that are currently '0'. If we can't, just keep '0'`s. @@ -5774,7 +5878,7 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, AVal = DAG.getBuildVector(VT, DL, AAmts); KVal = DAG.getBuildVector(ShVT, DL, KAmts); QVal = DAG.getBuildVector(VT, DL, QAmts); - } else if (VT.isScalableVector()) { + } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable " @@ -5784,6 +5888,7 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]); QVal = DAG.getSplatVector(VT, DL, QAmts[0]); } else { + assert(isa<ConstantSDNode>(D) && "Expected a constant"); PVal = PAmts[0]; AVal = AAmts[0]; KVal = KAmts[0]; @@ -5796,7 +5901,7 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, if (NeedToApplyOffset) { // We need ADD to do this. - if (!isOperationLegalOrCustom(ISD::ADD, VT)) + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) return SDValue(); // (add (mul N, P), A) @@ -5808,12 +5913,10 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // divisors as a performance improvement, since rotating by 0 is a no-op. if (HadEvenDivisor) { // We need ROTR to do this. - if (!isOperationLegalOrCustom(ISD::ROTR, VT)) + if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) return SDValue(); - SDNodeFlags Flags; - Flags.setExact(true); // SREM: (rotr (add (mul N, P), A), K) - Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags); + Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); Created.push_back(Op0.getNode()); } @@ -5831,10 +5934,13 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // we must fix-up results for said lanes. assert(VT.isVector() && "Can/should only get here for vectors."); + // NOTE: we avoid letting illegal types through even if we're before legalize + // ops – legalization has a hard time producing good code for the code that + // follows. if (!isOperationLegalOrCustom(ISD::SETEQ, VT) || !isOperationLegalOrCustom(ISD::AND, VT) || !isOperationLegalOrCustom(Cond, VT) || - !isOperationLegalOrCustom(ISD::VSELECT, VT)) + !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) return SDValue(); Created.push_back(Fold.getNode()); @@ -5860,8 +5966,8 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is // constant-folded, select can get lowered to a shuffle with constant mask. - SDValue Blended = - DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold); + SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin, + MaskedIsZero, Fold); return Blended; } @@ -5935,6 +6041,11 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, SDLoc DL(Op); + // Because getNegatedExpression can delete nodes we need a handle to keep + // temporary nodes alive in case the recursion manages to create an identical + // node. + std::list<HandleSDNode> Handles; + switch (Opcode) { case ISD::ConstantFP: { // Don't invert constant FP values after legalization unless the target says @@ -6003,11 +6114,18 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = CostX; @@ -6052,11 +6170,18 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y)) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = CostX; @@ -6094,15 +6219,25 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, if (!NegZ) break; + // Prevent this node from being deleted by the next two calls. + Handles.emplace_back(NegZ); + // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z)) NegatibleCost CostX = NegatibleCost::Expensive; SDValue NegX = getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth); + // Prevent this node from being deleted by the next call. + if (NegX) + Handles.emplace_back(NegX); + // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z)) NegatibleCost CostY = NegatibleCost::Expensive; SDValue NegY = getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth); + // We're done with the handles. + Handles.clear(); + // Negate the X if its cost is less or equal than Y. if (NegX && (CostX <= CostY)) { Cost = std::min(CostX, CostZ); @@ -6493,6 +6628,58 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps, return true; } +void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi, + SelectionDAG &DAG) const { + assert(Node->getNumOperands() == 3 && "Not a double-shift!"); + EVT VT = Node->getValueType(0); + unsigned VTBits = VT.getScalarSizeInBits(); + assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected"); + + bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS; + bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS; + SDValue ShOpLo = Node->getOperand(0); + SDValue ShOpHi = Node->getOperand(1); + SDValue ShAmt = Node->getOperand(2); + EVT ShAmtVT = ShAmt.getValueType(); + EVT ShAmtCCVT = + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT); + SDLoc dl(Node); + + // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and + // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized + // away during isel. + SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, + DAG.getConstant(VTBits - 1, dl, ShAmtVT)); + SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, + DAG.getConstant(VTBits - 1, dl, ShAmtVT)) + : DAG.getConstant(0, dl, VT); + + SDValue Tmp2, Tmp3; + if (IsSHL) { + Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); + } else { + Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); + Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); + } + + // If the shift amount is larger or equal than the width of a part we don't + // use the result from the FSHL/FSHR. Insert a test and select the appropriate + // values for large shift amounts. + SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, + DAG.getConstant(VTBits, dl, ShAmtVT)); + SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode, + DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE); + + if (IsSHL) { + Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); + Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); + } else { + Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); + Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); + } +} + bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const { unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; @@ -6514,7 +6701,7 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, // Expand f32 -> i64 conversion // This algorithm comes from compiler-rt's implementation of fixsfdi: - // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c + // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c unsigned SrcEltBits = SrcVT.getScalarSizeInBits(); EVT IntVT = SrcVT.changeTypeToInteger(); EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout()); @@ -7006,6 +7193,129 @@ bool TargetLowering::expandABS(SDNode *N, SDValue &Result, return true; } +SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { + SDLoc dl(N); + EVT VT = N->getValueType(0); + SDValue Op = N->getOperand(0); + + if (!VT.isSimple()) + return SDValue(); + + EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); + SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; + switch (VT.getSimpleVT().getScalarType().SimpleTy) { + default: + return SDValue(); + case MVT::i16: + // Use a rotate by 8. This can be further expanded if necessary. + return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); + case MVT::i32: + Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); + Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, + DAG.getConstant(0xFF0000, dl, VT)); + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); + Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); + Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); + return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); + case MVT::i64: + Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); + Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); + Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); + Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); + Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); + Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); + Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, + DAG.getConstant(255ULL<<48, dl, VT)); + Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, + DAG.getConstant(255ULL<<40, dl, VT)); + Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, + DAG.getConstant(255ULL<<32, dl, VT)); + Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, + DAG.getConstant(255ULL<<24, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, + DAG.getConstant(255ULL<<16, dl, VT)); + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, + DAG.getConstant(255ULL<<8 , dl, VT)); + Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); + Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); + Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); + Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); + Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); + Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); + return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); + } +} + +SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const { + SDLoc dl(N); + EVT VT = N->getValueType(0); + SDValue Op = N->getOperand(0); + EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); + unsigned Sz = VT.getScalarSizeInBits(); + + SDValue Tmp, Tmp2, Tmp3; + + // If we can, perform BSWAP first and then the mask+swap the i4, then i2 + // and finally the i1 pairs. + // TODO: We can easily support i4/i2 legal types if any target ever does. + if (Sz >= 8 && isPowerOf2_32(Sz)) { + // Create the masks - repeating the pattern every byte. + APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); + APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); + APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); + APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); + APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); + APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); + + // BSWAP if the type is wider than a single byte. + Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); + + // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + + // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + + // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + return Tmp; + } + + Tmp = DAG.getConstant(0, dl, VT); + for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { + if (I < J) + Tmp2 = + DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); + else + Tmp2 = + DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); + + APInt Shift(Sz, 1); + Shift <<= J; + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); + } + + return Tmp; +} + std::pair<SDValue, SDValue> TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const { @@ -7490,39 +7800,51 @@ TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment); } -static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, - SDValue Idx, - EVT VecVT, - const SDLoc &dl) { +static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, + EVT VecVT, const SDLoc &dl, + unsigned NumSubElts) { if (!VecVT.isScalableVector() && isa<ConstantSDNode>(Idx)) return Idx; EVT IdxVT = Idx.getValueType(); unsigned NElts = VecVT.getVectorMinNumElements(); if (VecVT.isScalableVector()) { - SDValue VS = DAG.getVScale(dl, IdxVT, - APInt(IdxVT.getFixedSizeInBits(), - NElts)); - SDValue Sub = DAG.getNode(ISD::SUB, dl, IdxVT, VS, - DAG.getConstant(1, dl, IdxVT)); - + // If this is a constant index and we know the value plus the number of the + // elements in the subvector minus one is less than the minimum number of + // elements then it's safe to return Idx. + if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx)) + if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts) + return Idx; + SDValue VS = + DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts)); + unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT; + SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS, + DAG.getConstant(NumSubElts, dl, IdxVT)); return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub); - } else { - if (isPowerOf2_32(NElts)) { - APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), - Log2_32(NElts)); - return DAG.getNode(ISD::AND, dl, IdxVT, Idx, - DAG.getConstant(Imm, dl, IdxVT)); - } } - + if (isPowerOf2_32(NElts) && NumSubElts == 1) { + APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts)); + return DAG.getNode(ISD::AND, dl, IdxVT, Idx, + DAG.getConstant(Imm, dl, IdxVT)); + } + unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0; return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, - DAG.getConstant(NElts - 1, dl, IdxVT)); + DAG.getConstant(MaxIndex, dl, IdxVT)); } SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const { + return getVectorSubVecPointer( + DAG, VecPtr, VecVT, + EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1), + Index); +} + +SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, + SDValue VecPtr, EVT VecVT, + EVT SubVecVT, + SDValue Index) const { SDLoc dl(Index); // Make sure the index type is big enough to compute in. Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType()); @@ -7534,7 +7856,13 @@ SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG, assert(EltSize * 8 == EltVT.getFixedSizeInBits() && "Converting bits to bytes lost precision"); - Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl); + // Scalable vectors don't need clamping as these are checked at compile time + if (SubVecVT.isFixedLengthVector()) { + assert(SubVecVT.getVectorElementType() == EltVT && + "Sub-vector must be a fixed vector with matching element type"); + Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl, + SubVecVT.getVectorNumElements()); + } EVT IdxVT = Index.getValueType(); @@ -7717,8 +8045,7 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { unsigned BitWidth = LHS.getScalarValueSizeInBits(); EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); - SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), - LHS, RHS); + SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); SDValue SumDiff = Result.getValue(0); SDValue Overflow = Result.getValue(1); SDValue Zero = DAG.getConstant(0, dl, VT); @@ -7732,7 +8059,9 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { } // Overflow ? 0xffff.... : (LHS + RHS) return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); - } else if (Opcode == ISD::USUBSAT) { + } + + if (Opcode == ISD::USUBSAT) { if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { // (LHS - RHS) & ~OverflowMask SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); @@ -7741,17 +8070,17 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { } // Overflow ? 0 : (LHS - RHS) return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); - } else { - // SatMax -> Overflow && SumDiff < 0 - // SatMin -> Overflow && SumDiff >= 0 - APInt MinVal = APInt::getSignedMinValue(BitWidth); - APInt MaxVal = APInt::getSignedMaxValue(BitWidth); - SDValue SatMin = DAG.getConstant(MinVal, dl, VT); - SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); - SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT); - Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin); - return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); } + + // SatMax -> Overflow && SumDiff < 0 + // SatMin -> Overflow && SumDiff >= 0 + APInt MinVal = APInt::getSignedMinValue(BitWidth); + APInt MaxVal = APInt::getSignedMaxValue(BitWidth); + SDValue SatMin = DAG.getConstant(MinVal, dl, VT); + SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); + SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT); + Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin); + return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); } SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const { @@ -8309,7 +8638,8 @@ bool TargetLowering::expandREM(SDNode *Node, SDValue &Result, SDVTList VTs = DAG.getVTList(VT, VT); Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1); return true; - } else if (isOperationLegalOrCustom(DivOpc, VT)) { + } + if (isOperationLegalOrCustom(DivOpc, VT)) { // X % Y -> X-X/Y*Y SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor); SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); @@ -8329,7 +8659,8 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node, EVT SrcVT = Src.getValueType(); EVT DstVT = Node->getValueType(0); - unsigned SatWidth = Node->getConstantOperandVal(1); + EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); + unsigned SatWidth = SatVT.getScalarSizeInBits(); unsigned DstWidth = DstVT.getScalarSizeInBits(); assert(SatWidth <= DstWidth && "Expected saturation width smaller than result width"); @@ -8420,3 +8751,210 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node, SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO); } + +SDValue TargetLowering::expandVectorSplice(SDNode *Node, + SelectionDAG &DAG) const { + assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!"); + assert(Node->getValueType(0).isScalableVector() && + "Fixed length vector types expected to use SHUFFLE_VECTOR!"); + + EVT VT = Node->getValueType(0); + SDValue V1 = Node->getOperand(0); + SDValue V2 = Node->getOperand(1); + int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue(); + SDLoc DL(Node); + + // Expand through memory thusly: + // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr + // Store V1, Ptr + // Store V2, Ptr + sizeof(V1) + // If (Imm < 0) + // TrailingElts = -Imm + // Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt)) + // else + // Ptr = Ptr + (Imm * sizeof(VT.Elt)) + // Res = Load Ptr + + Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false); + + EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), + VT.getVectorElementCount() * 2); + SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment); + EVT PtrVT = StackPtr.getValueType(); + auto &MF = DAG.getMachineFunction(); + auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); + auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); + + // Store the lo part of CONCAT_VECTORS(V1, V2) + SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo); + // Store the hi part of CONCAT_VECTORS(V1, V2) + SDValue OffsetToV2 = DAG.getVScale( + DL, PtrVT, + APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize())); + SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2); + SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo); + + if (Imm >= 0) { + // Load back the required element. getVectorElementPointer takes care of + // clamping the index if it's out-of-bounds. + StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2)); + // Load the spliced result + return DAG.getLoad(VT, DL, StoreV2, StackPtr, + MachinePointerInfo::getUnknownStack(MF)); + } + + uint64_t TrailingElts = -Imm; + + // NOTE: TrailingElts must be clamped so as not to read outside of V1:V2. + TypeSize EltByteSize = VT.getVectorElementType().getStoreSize(); + SDValue TrailingBytes = + DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT); + + if (TrailingElts > VT.getVectorMinNumElements()) { + SDValue VLBytes = DAG.getVScale( + DL, PtrVT, + APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize())); + TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes); + } + + // Calculate the start address of the spliced result. + StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes); + + // Load the spliced result + return DAG.getLoad(VT, DL, StoreV2, StackPtr2, + MachinePointerInfo::getUnknownStack(MF)); +} + +bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, + SDValue &LHS, SDValue &RHS, + SDValue &CC, bool &NeedInvert, + const SDLoc &dl, SDValue &Chain, + bool IsSignaling) const { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + MVT OpVT = LHS.getSimpleValueType(); + ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); + NeedInvert = false; + switch (TLI.getCondCodeAction(CCCode, OpVT)) { + default: + llvm_unreachable("Unknown condition code action!"); + case TargetLowering::Legal: + // Nothing to do. + break; + case TargetLowering::Expand: { + ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); + if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { + std::swap(LHS, RHS); + CC = DAG.getCondCode(InvCC); + return true; + } + // Swapping operands didn't work. Try inverting the condition. + bool NeedSwap = false; + InvCC = getSetCCInverse(CCCode, OpVT); + if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { + // If inverting the condition is not enough, try swapping operands + // on top of it. + InvCC = ISD::getSetCCSwappedOperands(InvCC); + NeedSwap = true; + } + if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { + CC = DAG.getCondCode(InvCC); + NeedInvert = true; + if (NeedSwap) + std::swap(LHS, RHS); + return true; + } + + ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; + unsigned Opc = 0; + switch (CCCode) { + default: + llvm_unreachable("Don't know how to expand this condition!"); + case ISD::SETUO: + if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) { + CC1 = ISD::SETUNE; + CC2 = ISD::SETUNE; + Opc = ISD::OR; + break; + } + assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && + "If SETUE is expanded, SETOEQ or SETUNE must be legal!"); + NeedInvert = true; + LLVM_FALLTHROUGH; + case ISD::SETO: + assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && + "If SETO is expanded, SETOEQ must be legal!"); + CC1 = ISD::SETOEQ; + CC2 = ISD::SETOEQ; + Opc = ISD::AND; + break; + case ISD::SETONE: + case ISD::SETUEQ: + // If the SETUO or SETO CC isn't legal, we might be able to use + // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one + // of SETOGT/SETOLT to be legal, the other can be emulated by swapping + // the operands. + CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; + if (!TLI.isCondCodeLegal(CC2, OpVT) && + (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) || + TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) { + CC1 = ISD::SETOGT; + CC2 = ISD::SETOLT; + Opc = ISD::OR; + NeedInvert = ((unsigned)CCCode & 0x8U); + break; + } + LLVM_FALLTHROUGH; + case ISD::SETOEQ: + case ISD::SETOGT: + case ISD::SETOGE: + case ISD::SETOLT: + case ISD::SETOLE: + case ISD::SETUNE: + case ISD::SETUGT: + case ISD::SETUGE: + case ISD::SETULT: + case ISD::SETULE: + // If we are floating point, assign and break, otherwise fall through. + if (!OpVT.isInteger()) { + // We can use the 4th bit to tell if we are the unordered + // or ordered version of the opcode. + CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; + Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; + CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); + break; + } + // Fallthrough if we are unsigned integer. + LLVM_FALLTHROUGH; + case ISD::SETLE: + case ISD::SETGT: + case ISD::SETGE: + case ISD::SETLT: + case ISD::SETNE: + case ISD::SETEQ: + // If all combinations of inverting the condition and swapping operands + // didn't work then we have no means to expand the condition. + llvm_unreachable("Don't know how to expand this condition!"); + } + + SDValue SetCC1, SetCC2; + if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { + // If we aren't the ordered or unorder operation, + // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). + SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling); + SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling); + } else { + // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) + SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling); + SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling); + } + if (Chain) + Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), + SetCC2.getValue(1)); + LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); + RHS = SDValue(); + CC = SDValue(); + return true; + } + } + return false; +} |