diff options
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 438 |
1 files changed, 296 insertions, 142 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 52a71b91d93f..313e07b5fdd6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -24,6 +24,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/MemoryLocation.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -63,6 +66,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/Utils/SizeOpts.h" #include <algorithm> #include <cassert> #include <cstdint> @@ -352,9 +356,9 @@ ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { (OldG << 2)); // New L bit. } -ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { +static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { unsigned Operation = Op; - if (isInteger) + if (isIntegerLike) Operation ^= 7; // Flip L, G, E bits, but not U. else Operation ^= 15; // Flip all of the condition bits. @@ -365,6 +369,15 @@ ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { return ISD::CondCode(Operation); } +ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { + return getSetCCInverseImpl(Op, Type.isInteger()); +} + +ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, + bool isIntegerLike) { + return getSetCCInverseImpl(Op, isIntegerLike); +} + /// For an integer comparison, return 1 if the comparison is a signed operation /// and 2 if the result is an unsigned comparison. Return zero if the operation /// does not depend on the sign of the input (setne and seteq). @@ -385,7 +398,8 @@ static int isSignedOp(ISD::CondCode Opcode) { } ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, - bool IsInteger) { + EVT Type) { + bool IsInteger = Type.isInteger(); if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) // Cannot fold a signed integer setcc with an unsigned integer setcc. return ISD::SETCC_INVALID; @@ -405,7 +419,8 @@ ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, } ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, - bool IsInteger) { + EVT Type) { + bool IsInteger = Type.isInteger(); if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) // Cannot fold a signed setcc with an unsigned setcc. return ISD::SETCC_INVALID; @@ -1005,7 +1020,9 @@ SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) void SelectionDAG::init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - LegacyDivergenceAnalysis * Divergence) { + LegacyDivergenceAnalysis * Divergence, + ProfileSummaryInfo *PSIin, + BlockFrequencyInfo *BFIin) { MF = &NewMF; SDAGISelPass = PassPtr; ORE = &NewORE; @@ -1014,6 +1031,8 @@ void SelectionDAG::init(MachineFunction &NewMF, LibInfo = LibraryInfo; Context = &MF->getFunction().getContext(); DA = Divergence; + PSI = PSIin; + BFI = BFIin; } SelectionDAG::~SelectionDAG() { @@ -1023,6 +1042,11 @@ SelectionDAG::~SelectionDAG() { delete DbgInfo; } +bool SelectionDAG::shouldOptForSize() const { + return MF->getFunction().hasOptSize() || + llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI); +} + void SelectionDAG::allnodes_clear() { assert(&*AllNodes.begin() == &EntryNode); AllNodes.remove(AllNodes.begin()); @@ -1101,6 +1125,20 @@ SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); } +std::pair<SDValue, SDValue> +SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, + const SDLoc &DL, EVT VT) { + assert(!VT.bitsEq(Op.getValueType()) && + "Strict no-op FP extend/round not allowed."); + SDValue Res = + VT.bitsGT(Op.getValueType()) + ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) + : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, + {Chain, Op, getIntPtrConstant(0, DL)}); + + return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1)); +} + SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { return VT.bitsGT(Op.getValueType()) ? getNode(ISD::ANY_EXTEND, DL, VT, Op) : @@ -1279,7 +1317,9 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, } SDValue Result(N, 0); - if (VT.isVector()) + if (VT.isScalableVector()) + Result = getSplatVector(VT, DL, Result); + else if (VT.isVector()) Result = getSplatBuildVector(VT, DL, Result); return Result; @@ -1425,7 +1465,7 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, assert((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"); if (Alignment == 0) - Alignment = MF->getFunction().hasOptSize() + Alignment = shouldOptForSize() ? getDataLayout().getABITypeAlignment(C->getType()) : getDataLayout().getPrefTypeAlignment(C->getType()); unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; @@ -2379,9 +2419,10 @@ SDValue SelectionDAG::getSplatValue(SDValue V) { /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that /// is less than the element bit-width of the shift node, return it. -static const APInt *getValidShiftAmountConstant(SDValue V) { +static const APInt *getValidShiftAmountConstant(SDValue V, + const APInt &DemandedElts) { unsigned BitWidth = V.getScalarValueSizeInBits(); - if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) { + if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) { // Shifting more than the bitwidth is not valid. const APInt &ShAmt = SA->getAPIntValue(); if (ShAmt.ult(BitWidth)) @@ -2392,13 +2433,16 @@ static const APInt *getValidShiftAmountConstant(SDValue V) { /// If a SHL/SRA/SRL node has constant vector shift amounts that are all less /// than the element bit-width of the shift node, return the minimum value. -static const APInt *getValidMinimumShiftAmountConstant(SDValue V) { +static const APInt * +getValidMinimumShiftAmountConstant(SDValue V, const APInt &DemandedElts) { unsigned BitWidth = V.getScalarValueSizeInBits(); auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); if (!BV) return nullptr; const APInt *MinShAmt = nullptr; for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { + if (!DemandedElts[i]) + continue; auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); if (!SA) return nullptr; @@ -2413,6 +2457,32 @@ static const APInt *getValidMinimumShiftAmountConstant(SDValue V) { return MinShAmt; } +/// If a SHL/SRA/SRL node has constant vector shift amounts that are all less +/// than the element bit-width of the shift node, return the maximum value. +static const APInt * +getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) { + unsigned BitWidth = V.getScalarValueSizeInBits(); + auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1)); + if (!BV) + return nullptr; + const APInt *MaxShAmt = nullptr; + for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { + if (!DemandedElts[i]) + continue; + auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i)); + if (!SA) + return nullptr; + // Shifting more than the bitwidth is not valid. + const APInt &ShAmt = SA->getAPIntValue(); + if (ShAmt.uge(BitWidth)) + return nullptr; + if (MaxShAmt && MaxShAmt->uge(ShAmt)) + continue; + MaxShAmt = &ShAmt; + } + return MaxShAmt; +} + /// Determine which bits of Op are known to be either zero or one and return /// them in Known. For vectors, the known bits are those that are shared by /// every vector element. @@ -2784,37 +2854,60 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setBitsFrom(1); break; case ISD::SETCC: + case ISD::STRICT_FSETCC: + case ISD::STRICT_FSETCCS: { + unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; // If we know the result of a setcc has the top bits zero, use this info. - if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == + if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) Known.Zero.setBitsFrom(1); break; + } case ISD::SHL: - if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + + if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { unsigned Shift = ShAmt->getZExtValue(); Known.Zero <<= Shift; Known.One <<= Shift; // Low bits are known zero. Known.Zero.setLowBits(Shift); + break; } + + // No matter the shift amount, the trailing zeros will stay zero. + Known.Zero = APInt::getLowBitsSet(BitWidth, Known.countMinTrailingZeros()); + Known.One.clearAllBits(); + + // Minimum shift low bits are known zero. + if (const APInt *ShMinAmt = + getValidMinimumShiftAmountConstant(Op, DemandedElts)) + Known.Zero.setLowBits(ShMinAmt->getZExtValue()); break; case ISD::SRL: - if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + + if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { unsigned Shift = ShAmt->getZExtValue(); Known.Zero.lshrInPlace(Shift); Known.One.lshrInPlace(Shift); // High bits are known zero. Known.Zero.setHighBits(Shift); - } else if (const APInt *ShMinAmt = getValidMinimumShiftAmountConstant(Op)) { - // Minimum shift high bits are known zero. - Known.Zero.setHighBits(ShMinAmt->getZExtValue()); + break; } + + // No matter the shift amount, the leading zeros will stay zero. + Known.Zero = APInt::getHighBitsSet(BitWidth, Known.countMinLeadingZeros()); + Known.One.clearAllBits(); + + // Minimum shift high bits are known zero. + if (const APInt *ShMinAmt = + getValidMinimumShiftAmountConstant(Op, DemandedElts)) + Known.Zero.setHighBits(ShMinAmt->getZExtValue()); break; case ISD::SRA: - if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { + if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); unsigned Shift = ShAmt->getZExtValue(); // Sign extend known zero/one bit (else is unknown). @@ -3336,20 +3429,20 @@ SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0, KnownBits N0Known = computeKnownBits(N0); bool overflow; - (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow); + (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow); if (!overflow) return OFK_Never; } // mulhi + 1 never overflow if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && - (~N1Known.Zero & 0x01) == ~N1Known.Zero) + (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue()) return OFK_Never; if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) { KnownBits N0Known = computeKnownBits(N0); - if ((~N0Known.Zero & 0x01) == ~N0Known.Zero) + if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue()) return OFK_Never; } @@ -3550,25 +3643,26 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, Tmp = VTBits - SrcVT.getScalarSizeInBits(); return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; } - case ISD::SRA: - Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); - // SRA X, C -> adds C sign bits. - if (ConstantSDNode *C = - isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { - APInt ShiftVal = C->getAPIntValue(); - ShiftVal += Tmp; - Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue(); - } + Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); + // SRA X, C -> adds C sign bits. + if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) + Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits); + else if (const APInt *ShAmt = + getValidMinimumShiftAmountConstant(Op, DemandedElts)) + Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits); return Tmp; case ISD::SHL: - if (ConstantSDNode *C = - isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { - // shl destroys sign bits. - Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); - if (C->getAPIntValue().uge(VTBits) || // Bad shift. - C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out. - return Tmp - C->getZExtValue(); + if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { + // shl destroys sign bits, ensure it doesn't shift out all sign bits. + Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); + if (ShAmt->ult(Tmp)) + return Tmp - ShAmt->getZExtValue(); + } else if (const APInt *ShAmt = + getValidMaximumShiftAmountConstant(Op, DemandedElts)) { + Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); + if (ShAmt->ult(Tmp)) + return Tmp - ShAmt->getZExtValue(); } break; case ISD::AND: @@ -3648,11 +3742,15 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, return VTBits; break; case ISD::SETCC: + case ISD::STRICT_FSETCC: + case ISD::STRICT_FSETCCS: { + unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; // If setcc returns 0/-1, all bits are sign bits. - if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == + if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == TargetLowering::ZeroOrNegativeOneBooleanContent) return VTBits; break; + } case ISD::ROTL: case ISD::ROTR: if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { @@ -4648,11 +4746,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, if (OpOpcode == ISD::UNDEF) return getUNDEF(VT); - // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 - if ((getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && - OpOpcode == ISD::FSUB) - return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1), - Operand.getOperand(0), Flags); if (OpOpcode == ISD::FNEG) // --X -> X return Operand.getOperand(0); break; @@ -4689,46 +4782,46 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, return V; } -static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1, - const APInt &C2) { +static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1, + const APInt &C2) { switch (Opcode) { - case ISD::ADD: return std::make_pair(C1 + C2, true); - case ISD::SUB: return std::make_pair(C1 - C2, true); - case ISD::MUL: return std::make_pair(C1 * C2, true); - case ISD::AND: return std::make_pair(C1 & C2, true); - case ISD::OR: return std::make_pair(C1 | C2, true); - case ISD::XOR: return std::make_pair(C1 ^ C2, true); - case ISD::SHL: return std::make_pair(C1 << C2, true); - case ISD::SRL: return std::make_pair(C1.lshr(C2), true); - case ISD::SRA: return std::make_pair(C1.ashr(C2), true); - case ISD::ROTL: return std::make_pair(C1.rotl(C2), true); - case ISD::ROTR: return std::make_pair(C1.rotr(C2), true); - case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true); - case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true); - case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true); - case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true); - case ISD::SADDSAT: return std::make_pair(C1.sadd_sat(C2), true); - case ISD::UADDSAT: return std::make_pair(C1.uadd_sat(C2), true); - case ISD::SSUBSAT: return std::make_pair(C1.ssub_sat(C2), true); - case ISD::USUBSAT: return std::make_pair(C1.usub_sat(C2), true); + case ISD::ADD: return C1 + C2; + case ISD::SUB: return C1 - C2; + case ISD::MUL: return C1 * C2; + case ISD::AND: return C1 & C2; + case ISD::OR: return C1 | C2; + case ISD::XOR: return C1 ^ C2; + case ISD::SHL: return C1 << C2; + case ISD::SRL: return C1.lshr(C2); + case ISD::SRA: return C1.ashr(C2); + case ISD::ROTL: return C1.rotl(C2); + case ISD::ROTR: return C1.rotr(C2); + case ISD::SMIN: return C1.sle(C2) ? C1 : C2; + case ISD::SMAX: return C1.sge(C2) ? C1 : C2; + case ISD::UMIN: return C1.ule(C2) ? C1 : C2; + case ISD::UMAX: return C1.uge(C2) ? C1 : C2; + case ISD::SADDSAT: return C1.sadd_sat(C2); + case ISD::UADDSAT: return C1.uadd_sat(C2); + case ISD::SSUBSAT: return C1.ssub_sat(C2); + case ISD::USUBSAT: return C1.usub_sat(C2); case ISD::UDIV: if (!C2.getBoolValue()) break; - return std::make_pair(C1.udiv(C2), true); + return C1.udiv(C2); case ISD::UREM: if (!C2.getBoolValue()) break; - return std::make_pair(C1.urem(C2), true); + return C1.urem(C2); case ISD::SDIV: if (!C2.getBoolValue()) break; - return std::make_pair(C1.sdiv(C2), true); + return C1.sdiv(C2); case ISD::SREM: if (!C2.getBoolValue()) break; - return std::make_pair(C1.srem(C2), true); + return C1.srem(C2); } - return std::make_pair(APInt(1, 0), false); + return llvm::None; } SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, @@ -4736,12 +4829,10 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, const ConstantSDNode *C2) { if (C1->isOpaque() || C2->isOpaque()) return SDValue(); - - std::pair<APInt, bool> Folded = FoldValue(Opcode, C1->getAPIntValue(), - C2->getAPIntValue()); - if (!Folded.second) - return SDValue(); - return getConstant(Folded.first, DL, VT); + if (Optional<APInt> Folded = + FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue())) + return getConstant(Folded.getValue(), DL, VT); + return SDValue(); } SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, @@ -5228,8 +5319,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ element type of the vector."); - // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. - if (N1.isUndef()) + // Extract from an undefined value or using an undefined index is undefined. + if (N1.isUndef() || N2.isUndef()) return getUNDEF(VT); // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF @@ -5506,6 +5597,15 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) return getUNDEF(VT); + + // Undefined index can be assumed out-of-bounds, so that's UNDEF too. + if (N3.isUndef()) + return getUNDEF(VT); + + // If the inserted element is an UNDEF, just use the input vector. + if (N2.isUndef()) + return N1; + break; } case ISD::INSERT_SUBVECTOR: { @@ -5697,10 +5797,19 @@ static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, return SDValue(nullptr, 0); } -SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset, - const SDLoc &DL) { +SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, int64_t Offset, + const SDLoc &DL, + const SDNodeFlags Flags) { EVT VT = Base.getValueType(); - return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT)); + return getMemBasePlusOffset(Base, getConstant(Offset, DL, VT), DL, Flags); +} + +SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, + const SDLoc &DL, + const SDNodeFlags Flags) { + assert(Offset.getValueType().isInteger()); + EVT BasePtrVT = Ptr.getValueType(); + return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); } /// Returns true if memcpy source is constant data. @@ -5722,12 +5831,13 @@ static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { SrcDelta + G->getOffset()); } -static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { +static bool shouldLowerMemFuncForSize(const MachineFunction &MF, + SelectionDAG &DAG) { // On Darwin, -Os means optimize for size without hurting performance, so // only really optimize for size when -Oz (MinSize) is used. if (MF.getTarget().getTargetTriple().isOSDarwin()) return MF.getFunction().hasMinSize(); - return MF.getFunction().hasOptSize(); + return DAG.shouldOptForSize(); } static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, @@ -5777,7 +5887,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, bool DstAlignCanChange = false; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); - bool OptSize = shouldLowerMemFuncForSize(MF); + bool OptSize = shouldLowerMemFuncForSize(MF, DAG); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -5960,7 +6070,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, bool DstAlignCanChange = false; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); - bool OptSize = shouldLowerMemFuncForSize(MF); + bool OptSize = shouldLowerMemFuncForSize(MF, DAG); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -6066,7 +6176,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, bool DstAlignCanChange = false; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); - bool OptSize = shouldLowerMemFuncForSize(MF); + bool OptSize = shouldLowerMemFuncForSize(MF, DAG); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -6557,7 +6667,9 @@ SDValue SelectionDAG::getMemIntrinsicNode( if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); - if (!Size) + if (!Size && MemVT.isScalableVector()) + Size = MemoryLocation::UnknownSize; + else if (!Size) Size = MemVT.getStoreSize(); MachineFunction &MF = getMachineFunction(); @@ -6951,16 +7063,22 @@ SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, } SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, - SDValue Ptr, SDValue Mask, SDValue PassThru, - EVT MemVT, MachineMemOperand *MMO, + SDValue Base, SDValue Offset, SDValue Mask, + SDValue PassThru, EVT MemVT, + MachineMemOperand *MMO, + ISD::MemIndexedMode AM, ISD::LoadExtType ExtTy, bool isExpanding) { - SDVTList VTs = getVTList(VT, MVT::Other); - SDValue Ops[] = { Chain, Ptr, Mask, PassThru }; + bool Indexed = AM != ISD::UNINDEXED; + assert((Indexed || Offset.isUndef()) && + "Unindexed masked load with an offset!"); + SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) + : getVTList(VT, MVT::Other); + SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); ID.AddInteger(MemVT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>( - dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO)); + dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -6968,7 +7086,7 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, return SDValue(E, 0); } auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, - ExtTy, isExpanding, MemVT, MMO); + AM, ExtTy, isExpanding, MemVT, MMO); createOperands(N, Ops); CSEMap.InsertNode(N, IP); @@ -6978,27 +7096,45 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, return V; } +SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, + SDValue Base, SDValue Offset, + ISD::MemIndexedMode AM) { + MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad); + assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!"); + return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base, + Offset, LD->getMask(), LD->getPassThru(), + LD->getMemoryVT(), LD->getMemOperand(), AM, + LD->getExtensionType(), LD->isExpandingLoad()); +} + SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, - SDValue Val, SDValue Ptr, SDValue Mask, - EVT MemVT, MachineMemOperand *MMO, - bool IsTruncating, bool IsCompressing) { + SDValue Val, SDValue Base, SDValue Offset, + SDValue Mask, EVT MemVT, + MachineMemOperand *MMO, + ISD::MemIndexedMode AM, bool IsTruncating, + bool IsCompressing) { assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); - SDVTList VTs = getVTList(MVT::Other); - SDValue Ops[] = { Chain, Val, Ptr, Mask }; + bool Indexed = AM != ISD::UNINDEXED; + assert((Indexed || Offset.isUndef()) && + "Unindexed masked store with an offset!"); + SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) + : getVTList(MVT::Other); + SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); ID.AddInteger(MemVT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>( - dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO)); + dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, - IsTruncating, IsCompressing, MemVT, MMO); + auto *N = + newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, + IsTruncating, IsCompressing, MemVT, MMO); createOperands(N, Ops); CSEMap.InsertNode(N, IP); @@ -7008,6 +7144,17 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, return V; } +SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, + SDValue Base, SDValue Offset, + ISD::MemIndexedMode AM) { + MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore); + assert(ST->getOffset().isUndef() && + "Masked store is already a indexed store!"); + return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset, + ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(), + AM, ST->isTruncatingStore(), ST->isCompressingStore()); +} + SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, @@ -7263,8 +7410,40 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, if (VTList.NumVTs == 1) return getNode(Opcode, DL, VTList.VTs[0], Ops); -#if 0 switch (Opcode) { + case ISD::STRICT_FP_EXTEND: + assert(VTList.NumVTs == 2 && Ops.size() == 2 && + "Invalid STRICT_FP_EXTEND!"); + assert(VTList.VTs[0].isFloatingPoint() && + Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"); + assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && + "STRICT_FP_EXTEND result type should be vector iff the operand " + "type is vector!"); + assert((!VTList.VTs[0].isVector() || + VTList.VTs[0].getVectorNumElements() == + Ops[1].getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); + assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && + "Invalid fpext node, dst <= src!"); + break; + case ISD::STRICT_FP_ROUND: + assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!"); + assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && + "STRICT_FP_ROUND result type should be vector iff the operand " + "type is vector!"); + assert((!VTList.VTs[0].isVector() || + VTList.VTs[0].getVectorNumElements() == + Ops[1].getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); + assert(VTList.VTs[0].isFloatingPoint() && + Ops[1].getValueType().isFloatingPoint() && + VTList.VTs[0].bitsLT(Ops[1].getValueType()) && + isa<ConstantSDNode>(Ops[2]) && + (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || + cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && + "Invalid STRICT_FP_ROUND!"); + break; +#if 0 // FIXME: figure out how to safely handle things like // int foo(int x) { return 1 << (x & 255); } // int bar() { return foo(256); } @@ -7283,8 +7462,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); } break; - } #endif + } // Memoize the node unless it returns a flag. SDNode *N; @@ -7740,38 +7919,11 @@ SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { switch (OrigOpc) { default: llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); - case ISD::STRICT_FADD: NewOpc = ISD::FADD; break; - case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break; - case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break; - case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break; - case ISD::STRICT_FREM: NewOpc = ISD::FREM; break; - case ISD::STRICT_FMA: NewOpc = ISD::FMA; break; - case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; break; - case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break; - case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break; - case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; break; - case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; break; - case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; break; - case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; break; - case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; break; - case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; break; - case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; break; - case ISD::STRICT_LRINT: NewOpc = ISD::LRINT; break; - case ISD::STRICT_LLRINT: NewOpc = ISD::LLRINT; break; - case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; break; - case ISD::STRICT_FNEARBYINT: NewOpc = ISD::FNEARBYINT; break; - case ISD::STRICT_FMAXNUM: NewOpc = ISD::FMAXNUM; break; - case ISD::STRICT_FMINNUM: NewOpc = ISD::FMINNUM; break; - case ISD::STRICT_FCEIL: NewOpc = ISD::FCEIL; break; - case ISD::STRICT_FFLOOR: NewOpc = ISD::FFLOOR; break; - case ISD::STRICT_LROUND: NewOpc = ISD::LROUND; break; - case ISD::STRICT_LLROUND: NewOpc = ISD::LLROUND; break; - case ISD::STRICT_FROUND: NewOpc = ISD::FROUND; break; - case ISD::STRICT_FTRUNC: NewOpc = ISD::FTRUNC; break; - case ISD::STRICT_FP_ROUND: NewOpc = ISD::FP_ROUND; break; - case ISD::STRICT_FP_EXTEND: NewOpc = ISD::FP_EXTEND; break; - case ISD::STRICT_FP_TO_SINT: NewOpc = ISD::FP_TO_SINT; break; - case ISD::STRICT_FP_TO_UINT: NewOpc = ISD::FP_TO_UINT; break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; +#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; +#include "llvm/IR/ConstrainedOps.def" } assert(Node->getNumValues() == 2 && "Unexpected number of results!"); @@ -8051,9 +8203,9 @@ void SelectionDAG::transferDbgValues(SDValue From, SDValue To, Expr = *Fragment; } // Clone the SDDbgValue and move it to To. - SDDbgValue *Clone = - getDbgValue(Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), - Dbg->getDebugLoc(), Dbg->getOrder()); + SDDbgValue *Clone = getDbgValue( + Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), Dbg->getDebugLoc(), + std::max(ToNode->getIROrder(), Dbg->getOrder())); ClonedDVs.push_back(Clone); if (InvalidateDbg) { @@ -8831,7 +8983,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely. - assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); + // TODO: Make MachineMemOperands aware of scalable vectors. + assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() && + "Size mismatch!"); } /// Profile - Gather unique data for the node. @@ -9245,11 +9399,11 @@ bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, /// it cannot be inferred. unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { // If this is a GlobalAddress + cst, return the alignment. - const GlobalValue *GV; + const GlobalValue *GV = nullptr; int64_t GVOffset = 0; if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { - unsigned IdxWidth = getDataLayout().getIndexTypeSizeInBits(GV->getType()); - KnownBits Known(IdxWidth); + unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); + KnownBits Known(PtrWidth); llvm::computeKnownBits(GV, Known, getDataLayout()); unsigned AlignBits = Known.countMinTrailingZeros(); unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; |