aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp881
1 files changed, 661 insertions, 220 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2090762e2ff4..2a98464425c4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -146,6 +146,10 @@ bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize);
return true;
}
+ if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
+ SplatVal = Op0->getValueAPF().bitcastToAPInt().truncOrSelf(EltSize);
+ return true;
+ }
}
auto *BV = dyn_cast<BuildVectorSDNode>(N);
@@ -338,8 +342,9 @@ bool ISD::matchBinaryPredicate(
return Match(LHSCst, RHSCst);
// TODO: Add support for vector UNDEF cases?
- if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
- ISD::BUILD_VECTOR != RHS.getOpcode())
+ if (LHS.getOpcode() != RHS.getOpcode() ||
+ (LHS.getOpcode() != ISD::BUILD_VECTOR &&
+ LHS.getOpcode() != ISD::SPLAT_VECTOR))
return false;
EVT SVT = LHS.getValueType().getScalarType();
@@ -879,6 +884,17 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
DeallocateNode(N);
}
+void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
+ assert(!(V->isVariadic() && isParameter));
+ if (isParameter)
+ ByvalParmDbgValues.push_back(V);
+ else
+ DbgValues.push_back(V);
+ for (const SDNode *Node : V->getSDNodes())
+ if (Node)
+ DbgValMap[Node].push_back(V);
+}
+
void SDDbgInfo::erase(const SDNode *Node) {
DbgValMapType::iterator I = DbgValMap.find(Node);
if (I == DbgValMap.end())
@@ -932,12 +948,12 @@ static void VerifySDNode(SDNode *N) {
assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
"Wrong number of operands!");
EVT EltVT = N->getValueType(0).getVectorElementType();
- for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
- assert((I->getValueType() == EltVT ||
- (EltVT.isInteger() && I->getValueType().isInteger() &&
- EltVT.bitsLE(I->getValueType()))) &&
- "Wrong operand type!");
- assert(I->getValueType() == N->getOperand(0).getValueType() &&
+ for (const SDUse &Op : N->ops()) {
+ assert((Op.getValueType() == EltVT ||
+ (EltVT.isInteger() && Op.getValueType().isInteger() &&
+ EltVT.bitsLE(Op.getValueType()))) &&
+ "Wrong operand type!");
+ assert(Op.getValueType() == N->getOperand(0).getValueType() &&
"Operands must all have the same type");
}
break;
@@ -1372,6 +1388,22 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
const APInt &NewVal = Elt->getValue();
EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
+
+ // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
+ if (VT.isScalableVector()) {
+ assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
+ "Can only handle an even split!");
+ unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
+
+ SmallVector<SDValue, 2> ScalarParts;
+ for (unsigned i = 0; i != Parts; ++i)
+ ScalarParts.push_back(getConstant(
+ NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
+ ViaEltVT, isT, isO));
+
+ return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
+ }
+
unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
@@ -1381,11 +1413,10 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
SmallVector<SDValue, 2> EltParts;
- for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
+ for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
EltParts.push_back(getConstant(
- NewVal.lshr(i * ViaEltSizeInBits).zextOrTrunc(ViaEltSizeInBits), DL,
+ NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
ViaEltVT, isT, isO));
- }
// EltParts is currently in little endian order. If we actually want
// big-endian order then reverse it now.
@@ -1498,17 +1529,17 @@ SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
EVT EltVT = VT.getScalarType();
if (EltVT == MVT::f32)
return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
- else if (EltVT == MVT::f64)
+ if (EltVT == MVT::f64)
return getConstantFP(APFloat(Val), DL, VT, isTarget);
- else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
- EltVT == MVT::f16 || EltVT == MVT::bf16) {
+ if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
+ EltVT == MVT::f16 || EltVT == MVT::bf16) {
bool Ignored;
APFloat APF = APFloat(Val);
APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
&Ignored);
return getConstantFP(APF, DL, VT, isTarget);
- } else
- llvm_unreachable("Unsupported type in getConstantFP");
+ }
+ llvm_unreachable("Unsupported type in getConstantFP");
}
SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
@@ -1717,6 +1748,25 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
return SDValue(CondCodeNodes[Cond], 0);
}
+SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
+ APInt One(ResVT.getScalarSizeInBits(), 1);
+ return getStepVector(DL, ResVT, One);
+}
+
+SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
+ assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
+ if (ResVT.isScalableVector())
+ return getNode(
+ ISD::STEP_VECTOR, DL, ResVT,
+ getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
+
+ SmallVector<SDValue, 16> OpsStepConstants;
+ for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
+ OpsStepConstants.push_back(
+ getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
+ return getBuildVector(ResVT, DL, OpsStepConstants);
+}
+
/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
/// point at N1 to point at N2 and indices that point at N2 to point at N1.
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
@@ -1727,7 +1777,7 @@ static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
SDValue N2, ArrayRef<int> Mask) {
assert(VT.getVectorNumElements() == Mask.size() &&
- "Must have the same number of vector elements as mask elements!");
+ "Must have the same number of vector elements as mask elements!");
assert(VT == N1.getValueType() && VT == N2.getValueType() &&
"Invalid VECTOR_SHUFFLE");
@@ -2430,7 +2480,9 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
return true;
case ISD::ADD:
case ISD::SUB:
- case ISD::AND: {
+ case ISD::AND:
+ case ISD::XOR:
+ case ISD::OR: {
APInt UndefLHS, UndefRHS;
SDValue LHS = V.getOperand(0);
SDValue RHS = V.getOperand(1);
@@ -2439,8 +2491,9 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
UndefElts = UndefLHS | UndefRHS;
return true;
}
- break;
+ return false;
}
+ case ISD::ABS:
case ISD::TRUNCATE:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
@@ -2495,6 +2548,9 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
case ISD::EXTRACT_SUBVECTOR: {
// Offset the demanded elts by the subvector index.
SDValue Src = V.getOperand(0);
+ // We don't support scalable vectors at the moment.
+ if (Src.getValueType().isScalableVector())
+ return false;
uint64_t Idx = V.getConstantOperandVal(1);
unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
APInt UndefSrcElts;
@@ -2578,12 +2634,21 @@ SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
return SDValue();
}
-SDValue SelectionDAG::getSplatValue(SDValue V) {
+SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
int SplatIdx;
- if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
- return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V),
- SrcVector.getValueType().getScalarType(), SrcVector,
+ if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
+ EVT SVT = SrcVector.getValueType().getScalarType();
+ EVT LegalSVT = SVT;
+ if (LegalTypes && !TLI->isTypeLegal(SVT)) {
+ if (!SVT.isInteger())
+ return SDValue();
+ LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
+ if (LegalSVT.bitsLT(SVT))
+ return SDValue();
+ }
+ return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
getVectorIdxConstant(SplatIdx, SDLoc(V)));
+ }
return SDValue();
}
@@ -2791,8 +2856,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
unsigned NumSubVectors = Op.getNumOperands();
for (unsigned i = 0; i != NumSubVectors; ++i) {
- APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
- DemandedSub = DemandedSub.trunc(NumSubVectorElts);
+ APInt DemandedSub =
+ DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
if (!!DemandedSub) {
SDValue Sub = Op.getOperand(i);
Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
@@ -2888,8 +2953,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
Depth + 1);
unsigned Shifts = IsLE ? i : SubScale - 1 - i;
- Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
- Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
+ Known.insertBits(Known2, SubBitWidth * Shifts);
}
}
@@ -2913,8 +2977,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
if (DemandedElts[i]) {
unsigned Shifts = IsLE ? i : NumElts - 1 - i;
unsigned Offset = (Shifts % SubScale) * BitWidth;
- Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
- Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
+ Known = KnownBits::commonBits(Known,
+ Known2.extractBits(BitWidth, Offset));
// If we don't know any bits, early out.
if (Known.isUnknown())
break;
@@ -2943,7 +3007,39 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
case ISD::MUL: {
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
- Known = KnownBits::computeForMul(Known, Known2);
+ Known = KnownBits::mul(Known, Known2);
+ break;
+ }
+ case ISD::MULHU: {
+ Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+ Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+ Known = KnownBits::mulhu(Known, Known2);
+ break;
+ }
+ case ISD::MULHS: {
+ Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+ Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+ Known = KnownBits::mulhs(Known, Known2);
+ break;
+ }
+ case ISD::UMUL_LOHI: {
+ assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
+ Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+ Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+ if (Op.getResNo() == 0)
+ Known = KnownBits::mul(Known, Known2);
+ else
+ Known = KnownBits::mulhu(Known, Known2);
+ break;
+ }
+ case ISD::SMUL_LOHI: {
+ assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
+ Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+ Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+ if (Op.getResNo() == 0)
+ Known = KnownBits::mul(Known, Known2);
+ else
+ Known = KnownBits::mulhs(Known, Known2);
break;
}
case ISD::UDIV: {
@@ -2975,7 +3071,6 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
break;
case ISD::SMULO:
case ISD::UMULO:
- case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
if (Op.getResNo() != 1)
break;
// The boolean result conforms to getBooleanContents.
@@ -3373,6 +3468,12 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
Known = Known2.abs();
break;
}
+ case ISD::USUBSAT: {
+ // The result of usubsat will never be larger than the LHS.
+ Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+ Known.Zero.setHighBits(Known2.countMinLeadingZeros());
+ break;
+ }
case ISD::UMIN: {
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -3424,6 +3525,42 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
Known = KnownBits::smin(Known, Known2);
break;
}
+ case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
+ if (Op.getResNo() == 1) {
+ // The boolean result conforms to getBooleanContents.
+ // If we know the result of a setcc has the top bits zero, use this info.
+ // We know that we have an integer-based boolean since these operations
+ // are only available for integer.
+ if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
+ TargetLowering::ZeroOrOneBooleanContent &&
+ BitWidth > 1)
+ Known.Zero.setBitsFrom(1);
+ break;
+ }
+ LLVM_FALLTHROUGH;
+ case ISD::ATOMIC_CMP_SWAP:
+ case ISD::ATOMIC_SWAP:
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_LOAD: {
+ unsigned MemBits =
+ cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
+ // If we are looking at the loaded value.
+ if (Op.getResNo() == 0) {
+ if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
+ Known.Zero.setBitsFrom(MemBits);
+ }
+ break;
+ }
case ISD::FrameIndex:
case ISD::TargetFrameIndex:
TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
@@ -3867,6 +4004,12 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
(VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
}
+ case ISD::SREM:
+ // The sign bit is the LHS's sign bit, except when the result of the
+ // remainder is zero. The magnitude of the result should be less than or
+ // equal to the magnitude of the LHS. Therefore, the result should have
+ // at least as many sign bits as the left hand side.
+ return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
case ISD::TRUNCATE: {
// Check if the sign bits of source go down as far as the truncated value.
unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
@@ -3922,6 +4065,9 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
SDValue InVec = Op.getOperand(0);
SDValue EltNo = Op.getOperand(1);
EVT VecVT = InVec.getValueType();
+ // ComputeNumSignBits not yet implemented for scalable vectors.
+ if (VecVT.isScalableVector())
+ break;
const unsigned BitWidth = Op.getValueSizeInBits();
const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
const unsigned NumSrcElts = VecVT.getVectorNumElements();
@@ -3961,8 +4107,8 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
unsigned NumSubVectors = Op.getNumOperands();
for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
- APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
- DemandedSub = DemandedSub.trunc(NumSubVectorElts);
+ APInt DemandedSub =
+ DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
if (!DemandedSub)
continue;
Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
@@ -3995,6 +4141,33 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
return Tmp;
}
+ case ISD::ATOMIC_CMP_SWAP:
+ case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
+ case ISD::ATOMIC_SWAP:
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_LOAD: {
+ Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
+ // If we are looking at the loaded value.
+ if (Op.getResNo() == 0) {
+ if (Tmp == VTBits)
+ return 1; // early-out
+ if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
+ return VTBits - Tmp + 1;
+ if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
+ return VTBits - Tmp;
+ }
+ break;
+ }
}
// If we are looking at the loaded value of the SDNode.
@@ -4075,6 +4248,61 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
return std::max(FirstAnswer, Mask.countLeadingOnes());
}
+bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
+ unsigned Depth) const {
+ // Early out for FREEZE.
+ if (Op.getOpcode() == ISD::FREEZE)
+ return true;
+
+ // TODO: Assume we don't know anything for now.
+ EVT VT = Op.getValueType();
+ if (VT.isScalableVector())
+ return false;
+
+ APInt DemandedElts = VT.isVector()
+ ? APInt::getAllOnesValue(VT.getVectorNumElements())
+ : APInt(1, 1);
+ return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
+}
+
+bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
+ const APInt &DemandedElts,
+ bool PoisonOnly,
+ unsigned Depth) const {
+ unsigned Opcode = Op.getOpcode();
+
+ // Early out for FREEZE.
+ if (Opcode == ISD::FREEZE)
+ return true;
+
+ if (Depth >= MaxRecursionDepth)
+ return false; // Limit search depth.
+
+ if (isIntOrFPConstant(Op))
+ return true;
+
+ switch (Opcode) {
+ case ISD::UNDEF:
+ return PoisonOnly;
+
+ // TODO: ISD::BUILD_VECTOR handling
+
+ // TODO: Search for noundef attributes from library functions.
+
+ // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
+
+ default:
+ // Allow the target to implement this method for its nodes.
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
+ Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
+ return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
+ Op, DemandedElts, *this, PoisonOnly, Depth);
+ break;
+ }
+
+ return false;
+}
+
bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
!isa<ConstantSDNode>(Op.getOperand(1)))
@@ -4256,7 +4484,16 @@ bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
assert(A.getValueType() == B.getValueType() &&
"Values must have the same type");
- return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
+ return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
+ computeKnownBits(B));
+}
+
+static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
+ SelectionDAG &DAG) {
+ if (cast<ConstantSDNode>(Step)->isNullValue())
+ return DAG.getConstant(0, DL, VT);
+
+ return SDValue();
}
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
@@ -4408,6 +4645,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue Operand, const SDNodeFlags Flags) {
+ assert(Operand.getOpcode() != ISD::DELETED_NODE &&
+ "Operand is DELETED_NODE!");
// Constant fold unary operations with an integer constant operand. Even
// opaque constant will be folded, because the folding of unary operations
// doesn't create new constants with different values. Nevertheless, the
@@ -4424,10 +4663,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (C->isOpaque())
break;
LLVM_FALLTHROUGH;
- case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
C->isTargetOpcode(), C->isOpaque());
+ case ISD::ANY_EXTEND:
+ // Some targets like RISCV prefer to sign extend some types.
+ if (TLI->isSExtCheaperThanZExt(Operand.getValueType(), VT))
+ return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
+ C->isTargetOpcode(), C->isOpaque());
+ return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
+ C->isTargetOpcode(), C->isOpaque());
case ISD::UINT_TO_FP:
case ISD::SINT_TO_FP: {
APFloat apf(EVTToAPFloatSemantics(VT),
@@ -4478,6 +4723,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
APFloat::rmNearestTiesToEven, &Ignored);
return getConstantFP(FPV, DL, VT);
}
+ case ISD::STEP_VECTOR: {
+ if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this))
+ return V;
+ break;
+ }
}
}
@@ -4531,9 +4781,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
case ISD::BITCAST:
if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
- else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
+ if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
+ return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
+ if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
- else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
+ if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
break;
case ISD::FP_TO_FP16: {
@@ -4548,45 +4800,48 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
}
// Constant fold unary operations with a vector integer or float operand.
- if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
- if (BV->isConstant()) {
- switch (Opcode) {
- default:
- // FIXME: Entirely reasonable to perform folding of other unary
- // operations here as the need arises.
- break;
- case ISD::FNEG:
- case ISD::FABS:
- case ISD::FCEIL:
- case ISD::FTRUNC:
- case ISD::FFLOOR:
- case ISD::FP_EXTEND:
- case ISD::FP_TO_SINT:
- case ISD::FP_TO_UINT:
- case ISD::TRUNCATE:
- case ISD::ANY_EXTEND:
- case ISD::ZERO_EXTEND:
- case ISD::SIGN_EXTEND:
- case ISD::UINT_TO_FP:
- case ISD::SINT_TO_FP:
- case ISD::ABS:
- case ISD::BITREVERSE:
- case ISD::BSWAP:
- case ISD::CTLZ:
- case ISD::CTLZ_ZERO_UNDEF:
- case ISD::CTTZ:
- case ISD::CTTZ_ZERO_UNDEF:
- case ISD::CTPOP: {
- SDValue Ops = { Operand };
- if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
- return Fold;
- }
- }
- }
+ switch (Opcode) {
+ default:
+ // FIXME: Entirely reasonable to perform folding of other unary
+ // operations here as the need arises.
+ break;
+ case ISD::FNEG:
+ case ISD::FABS:
+ case ISD::FCEIL:
+ case ISD::FTRUNC:
+ case ISD::FFLOOR:
+ case ISD::FP_EXTEND:
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ case ISD::TRUNCATE:
+ case ISD::ANY_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::SIGN_EXTEND:
+ case ISD::UINT_TO_FP:
+ case ISD::SINT_TO_FP:
+ case ISD::ABS:
+ case ISD::BITREVERSE:
+ case ISD::BSWAP:
+ case ISD::CTLZ:
+ case ISD::CTLZ_ZERO_UNDEF:
+ case ISD::CTTZ:
+ case ISD::CTTZ_ZERO_UNDEF:
+ case ISD::CTPOP: {
+ SDValue Ops = {Operand};
+ if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
+ return Fold;
+ }
}
unsigned OpOpcode = Operand.getNode()->getOpcode();
switch (Opcode) {
+ case ISD::STEP_VECTOR:
+ assert(VT.isScalableVector() &&
+ "STEP_VECTOR can only be used with scalable types");
+ assert(OpOpcode == ISD::TargetConstant &&
+ VT.getVectorElementType() == Operand.getValueType() &&
+ "Unexpected step operand");
+ break;
case ISD::FREEZE:
assert(VT == Operand.getValueType() && "Unexpected VT!");
break;
@@ -4641,7 +4896,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
"Invalid sext node, dst < src!");
if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
- else if (OpOpcode == ISD::UNDEF)
+ if (OpOpcode == ISD::UNDEF)
// sext(undef) = 0, because the top bits will all be the same.
return getConstant(0, DL, VT);
break;
@@ -4660,7 +4915,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
"Invalid zext node, dst < src!");
if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
- else if (OpOpcode == ISD::UNDEF)
+ if (OpOpcode == ISD::UNDEF)
// zext(undef) = 0, because the top bits will be zero.
return getConstant(0, DL, VT);
break;
@@ -4682,7 +4937,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
OpOpcode == ISD::ANY_EXTEND)
// (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
- else if (OpOpcode == ISD::UNDEF)
+ if (OpOpcode == ISD::UNDEF)
return getUNDEF(VT);
// (ext (trunc x)) -> x
@@ -4728,8 +4983,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
assert(VT.isVector() && "This DAG node is restricted to vector types.");
assert(Operand.getValueType().bitsLE(VT) &&
"The input must be the same size or smaller than the result.");
- assert(VT.getVectorNumElements() <
- Operand.getValueType().getVectorNumElements() &&
+ assert(VT.getVectorMinNumElements() <
+ Operand.getValueType().getVectorMinNumElements() &&
"The destination vector type must have fewer lanes than the input.");
break;
case ISD::ABS:
@@ -4879,6 +5134,18 @@ static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
if (!C2.getBoolValue())
break;
return C1.srem(C2);
+ case ISD::MULHS: {
+ unsigned FullWidth = C1.getBitWidth() * 2;
+ APInt C1Ext = C1.sext(FullWidth);
+ APInt C2Ext = C2.sext(FullWidth);
+ return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
+ }
+ case ISD::MULHU: {
+ unsigned FullWidth = C1.getBitWidth() * 2;
+ APInt C1Ext = C1.zext(FullWidth);
+ APInt C2Ext = C2.zext(FullWidth);
+ return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
+ }
}
return llvm::None;
}
@@ -4933,7 +5200,10 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
// If the opcode is a target-specific ISD node, there's nothing we can
// do here and the operand rules may not line up with the below, so
// bail early.
- if (Opcode >= ISD::BUILTIN_OP_END)
+ // We can't create a scalar CONCAT_VECTORS so skip it. It will break
+ // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
+ // foldCONCAT_VECTORS in getNode before this is called.
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
return SDValue();
// For now, the array Ops should only contain two values.
@@ -4973,27 +5243,20 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
return FoldSymbolOffset(Opcode, VT, GA, N1);
- // TODO: All the folds below are performed lane-by-lane and assume a fixed
- // vector width, however we should be able to do constant folds involving
- // splat vector nodes too.
- if (VT.isScalableVector())
- return SDValue();
-
// For fixed width vectors, extract each constant element and fold them
// individually. Either input may be an undef value.
- auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
- if (!BV1 && !N1->isUndef())
+ bool IsBVOrSV1 = N1->getOpcode() == ISD::BUILD_VECTOR ||
+ N1->getOpcode() == ISD::SPLAT_VECTOR;
+ if (!IsBVOrSV1 && !N1->isUndef())
return SDValue();
- auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
- if (!BV2 && !N2->isUndef())
+ bool IsBVOrSV2 = N2->getOpcode() == ISD::BUILD_VECTOR ||
+ N2->getOpcode() == ISD::SPLAT_VECTOR;
+ if (!IsBVOrSV2 && !N2->isUndef())
return SDValue();
// If both operands are undef, that's handled the same way as scalars.
- if (!BV1 && !BV2)
+ if (!IsBVOrSV1 && !IsBVOrSV2)
return SDValue();
- assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) &&
- "Vector binop with different number of elements in operands?");
-
EVT SVT = VT.getScalarType();
EVT LegalSVT = SVT;
if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
@@ -5001,19 +5264,46 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
if (LegalSVT.bitsLT(SVT))
return SDValue();
}
+
SmallVector<SDValue, 4> Outputs;
- unsigned NumOps = BV1 ? BV1->getNumOperands() : BV2->getNumOperands();
+ unsigned NumOps = 0;
+ if (IsBVOrSV1)
+ NumOps = std::max(NumOps, N1->getNumOperands());
+ if (IsBVOrSV2)
+ NumOps = std::max(NumOps, N2->getNumOperands());
+ assert(NumOps != 0 && "Expected non-zero operands");
+ // Scalable vectors should only be SPLAT_VECTOR or UNDEF here. We only need
+ // one iteration for that.
+ assert((!VT.isScalableVector() || NumOps == 1) &&
+ "Scalable vector should only have one scalar");
+
for (unsigned I = 0; I != NumOps; ++I) {
- SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT);
- SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT);
+ // We can have a fixed length SPLAT_VECTOR and a BUILD_VECTOR so we need
+ // to use operand 0 of the SPLAT_VECTOR for each fixed element.
+ SDValue V1;
+ if (N1->getOpcode() == ISD::BUILD_VECTOR)
+ V1 = N1->getOperand(I);
+ else if (N1->getOpcode() == ISD::SPLAT_VECTOR)
+ V1 = N1->getOperand(0);
+ else
+ V1 = getUNDEF(SVT);
+
+ SDValue V2;
+ if (N2->getOpcode() == ISD::BUILD_VECTOR)
+ V2 = N2->getOperand(I);
+ else if (N2->getOpcode() == ISD::SPLAT_VECTOR)
+ V2 = N2->getOperand(0);
+ else
+ V2 = getUNDEF(SVT);
+
if (SVT.isInteger()) {
- if (V1->getValueType(0).bitsGT(SVT))
+ if (V1.getValueType().bitsGT(SVT))
V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
- if (V2->getValueType(0).bitsGT(SVT))
+ if (V2.getValueType().bitsGT(SVT))
V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
}
- if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
+ if (V1.getValueType() != SVT || V2.getValueType() != SVT)
return SDValue();
// Fold one vector element.
@@ -5028,14 +5318,21 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
Outputs.push_back(ScalarResult);
}
- assert(VT.getVectorNumElements() == Outputs.size() &&
- "Vector size mismatch!");
+ if (N1->getOpcode() == ISD::BUILD_VECTOR ||
+ N2->getOpcode() == ISD::BUILD_VECTOR) {
+ assert(VT.getVectorNumElements() == Outputs.size() &&
+ "Vector size mismatch!");
+
+ // Build a big vector out of the scalar elements we generated.
+ return getBuildVector(VT, SDLoc(), Outputs);
+ }
- // We may have a vector type but a scalar result. Create a splat.
- Outputs.resize(VT.getVectorNumElements(), Outputs.back());
+ assert((N1->getOpcode() == ISD::SPLAT_VECTOR ||
+ N2->getOpcode() == ISD::SPLAT_VECTOR) &&
+ "One operand should be a splat vector");
- // Build a big vector out of the scalar elements we generated.
- return getBuildVector(VT, SDLoc(), Outputs);
+ assert(Outputs.size() == 1 && "Vector size mismatch!");
+ return getSplatVector(VT, SDLoc(), Outputs[0]);
}
// TODO: Merge with FoldConstantArithmetic
@@ -5056,30 +5353,26 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
if (!VT.isVector())
return SDValue();
- // TODO: All the folds below are performed lane-by-lane and assume a fixed
- // vector width, however we should be able to do constant folds involving
- // splat vector nodes too.
- if (VT.isScalableVector())
- return SDValue();
-
- // From this point onwards all vectors are assumed to be fixed width.
- unsigned NumElts = VT.getVectorNumElements();
+ ElementCount NumElts = VT.getVectorElementCount();
- auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
+ auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
return !Op.getValueType().isVector() ||
- Op.getValueType().getVectorNumElements() == NumElts;
+ Op.getValueType().getVectorElementCount() == NumElts;
};
- auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
+ auto IsConstantBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
+ APInt SplatVal;
BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
- return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
- (BV && BV->isConstant());
+ return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
+ (BV && BV->isConstant()) ||
+ (Op.getOpcode() == ISD::SPLAT_VECTOR &&
+ ISD::isConstantSplatVector(Op.getNode(), SplatVal));
};
// All operands must be vector types with the same number of elements as
// the result type and must be either UNDEF or a build vector of constant
// or UNDEF scalars.
- if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
+ if (!llvm::all_of(Ops, IsConstantBuildVectorSplatVectorOrUndef) ||
!llvm::all_of(Ops, IsScalarOrSameVectorSize))
return SDValue();
@@ -5096,14 +5389,19 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
return SDValue();
}
+ // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
+ // only have one operand to check. For fixed-length vector types we may have
+ // a combination of BUILD_VECTOR and SPLAT_VECTOR.
+ unsigned NumOperands = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
+
// Constant fold each scalar lane separately.
SmallVector<SDValue, 4> ScalarResults;
- for (unsigned i = 0; i != NumElts; i++) {
+ for (unsigned I = 0; I != NumOperands; I++) {
SmallVector<SDValue, 4> ScalarOps;
for (SDValue Op : Ops) {
EVT InSVT = Op.getValueType().getScalarType();
- BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
- if (!InBV) {
+ if (Op.getOpcode() != ISD::BUILD_VECTOR &&
+ Op.getOpcode() != ISD::SPLAT_VECTOR) {
// We've checked that this is UNDEF or a constant of some kind.
if (Op.isUndef())
ScalarOps.push_back(getUNDEF(InSVT));
@@ -5112,7 +5410,8 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
continue;
}
- SDValue ScalarOp = InBV->getOperand(i);
+ SDValue ScalarOp =
+ Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
EVT ScalarVT = ScalarOp.getValueType();
// Build vector (integer) scalar operands may need implicit
@@ -5137,7 +5436,8 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
ScalarResults.push_back(ScalarResult);
}
- SDValue V = getBuildVector(VT, DL, ScalarResults);
+ SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
+ : getBuildVector(VT, DL, ScalarResults);
NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
return V;
}
@@ -5243,6 +5543,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue N1, SDValue N2, const SDNodeFlags Flags) {
+ assert(N1.getOpcode() != ISD::DELETED_NODE &&
+ N2.getOpcode() != ISD::DELETED_NODE &&
+ "Operand is DELETED_NODE!");
ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
@@ -5304,14 +5607,19 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
// it's worth handling here.
if (N2C && N2C->isNullValue())
return N1;
+ if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
+ VT.getVectorElementType() == MVT::i1)
+ return getNode(ISD::XOR, DL, VT, N1, N2);
break;
case ISD::MUL:
assert(VT.isInteger() && "This operator does not apply to FP types!");
assert(N1.getValueType() == N2.getValueType() &&
N1.getValueType() == VT && "Binary operator types must match!");
+ if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
+ return getNode(ISD::AND, DL, VT, N1, N2);
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
- APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
- APInt N2CImm = N2C->getAPIntValue();
+ const APInt &MulImm = N1->getConstantOperandAPInt(0);
+ const APInt &N2CImm = N2C->getAPIntValue();
return getVScale(DL, VT, MulImm * N2CImm);
}
break;
@@ -5328,6 +5636,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
assert(VT.isInteger() && "This operator does not apply to FP types!");
assert(N1.getValueType() == N2.getValueType() &&
N1.getValueType() == VT && "Binary operator types must match!");
+ if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
+ // fold (add_sat x, y) -> (or x, y) for bool types.
+ if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
+ return getNode(ISD::OR, DL, VT, N1, N2);
+ // fold (sub_sat x, y) -> (and x, ~y) for bool types.
+ if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
+ return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
+ }
break;
case ISD::SMIN:
case ISD::UMAX:
@@ -5364,8 +5680,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
break;
case ISD::SHL:
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
- APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
- APInt ShiftImm = N2C->getAPIntValue();
+ const APInt &MulImm = N1->getConstantOperandAPInt(0);
+ const APInt &ShiftImm = N2C->getAPIntValue();
return getVScale(DL, VT, MulImm << ShiftImm);
}
LLVM_FALLTHROUGH;
@@ -5444,6 +5760,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
const APInt &Val = N1C->getAPIntValue();
return SignExtendInReg(Val, VT);
}
+
if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
SmallVector<SDValue, 8> Ops;
llvm::EVT OpVT = N1.getOperand(0).getValueType();
@@ -5461,6 +5778,22 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
}
break;
}
+ case ISD::FP_TO_SINT_SAT:
+ case ISD::FP_TO_UINT_SAT: {
+ assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
+ N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
+ assert(N1.getValueType().isVector() == VT.isVector() &&
+ "FP_TO_*INT_SAT type should be vector iff the operand type is "
+ "vector!");
+ assert((!VT.isVector() || VT.getVectorNumElements() ==
+ N1.getValueType().getVectorNumElements()) &&
+ "Vector element counts must match in FP_TO_*INT_SAT");
+ assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
+ "Type to saturate to must be a scalar.");
+ assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
+ "Not extending!");
+ break;
+ }
case ISD::EXTRACT_VECTOR_ELT:
assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
"The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
@@ -5523,10 +5856,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
if (VT == N1.getOperand(1).getValueType())
return N1.getOperand(1);
- else
- return getSExtOrTrunc(N1.getOperand(1), DL, VT);
+ return getSExtOrTrunc(N1.getOperand(1), DL, VT);
}
-
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
}
}
@@ -5563,11 +5894,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (N1C) {
unsigned ElementSize = VT.getSizeInBits();
unsigned Shift = ElementSize * N2C->getZExtValue();
- APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
- return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
+ const APInt &Val = N1C->getAPIntValue();
+ return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
}
break;
- case ISD::EXTRACT_SUBVECTOR:
+ case ISD::EXTRACT_SUBVECTOR: {
EVT N1VT = N1.getValueType();
assert(VT.isVector() && N1VT.isVector() &&
"Extract subvector VTs must be vectors!");
@@ -5584,9 +5915,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
N1VT.getVectorMinNumElements()) &&
"Extract subvector overflow!");
assert(N2C->getAPIntValue().getBitWidth() ==
- TLI->getVectorIdxTy(getDataLayout())
- .getSizeInBits()
- .getFixedSize() &&
+ TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
"Constant index for EXTRACT_SUBVECTOR has an invalid size");
// Trivial extraction.
@@ -5612,6 +5941,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
return N1.getOperand(1);
break;
}
+ }
// Perform trivial constant folding.
if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
@@ -5707,6 +6037,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue N1, SDValue N2, SDValue N3,
const SDNodeFlags Flags) {
+ assert(N1.getOpcode() != ISD::DELETED_NODE &&
+ N2.getOpcode() != ISD::DELETED_NODE &&
+ N3.getOpcode() != ISD::DELETED_NODE &&
+ "Operand is DELETED_NODE!");
// Perform various simplifications.
switch (Opcode) {
case ISD::FMA: {
@@ -5806,6 +6140,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
cast<ConstantSDNode>(N3)->getZExtValue()) <=
VT.getVectorMinNumElements()) &&
"Insert subvector overflow!");
+ assert(cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() ==
+ TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
+ "Constant index for INSERT_SUBVECTOR has an invalid size");
// Trivial insertion.
if (VT == N2VT)
@@ -5939,17 +6276,17 @@ static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
if (Slice.Array == nullptr) {
if (VT.isInteger())
return DAG.getConstant(0, dl, VT);
- else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
+ if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
return DAG.getConstantFP(0.0, dl, VT);
- else if (VT.isVector()) {
+ if (VT.isVector()) {
unsigned NumElts = VT.getVectorNumElements();
MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
return DAG.getNode(ISD::BITCAST, dl, VT,
DAG.getConstant(0, dl,
EVT::getVectorVT(*DAG.getContext(),
EltVT, NumElts)));
- } else
- llvm_unreachable("Expected type!");
+ }
+ llvm_unreachable("Expected type!");
}
assert(!VT.isVector() && "Can't handle vector type here!");
@@ -6056,7 +6393,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
uint64_t Size, Align Alignment,
bool isVol, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo) {
+ MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo) {
// Turn a memcpy of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -6103,7 +6441,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
// Don't promote to an alignment that would require dynamic stack
// realignment.
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
- if (!TRI->needsStackRealignment(MF))
+ if (!TRI->hasStackRealignment(MF))
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
NewAlign = NewAlign / 2;
@@ -6115,6 +6453,10 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
}
}
+ // Prepare AAInfo for loads/stores after lowering this memcpy.
+ AAMDNodes NewAAInfo = AAInfo;
+ NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
+
MachineMemOperand::Flags MMOFlags =
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
SmallVector<SDValue, 16> OutLoadChains;
@@ -6157,7 +6499,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
Store = DAG.getStore(
Chain, dl, Value,
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
- DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags);
+ DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
OutChains.push_back(Store);
}
}
@@ -6181,13 +6523,13 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
ISD::EXTLOAD, dl, NVT, Chain,
DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
SrcPtrInfo.getWithOffset(SrcOff), VT,
- commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags);
+ commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
OutLoadChains.push_back(Value.getValue(1));
Store = DAG.getTruncStore(
Chain, dl, Value,
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
- DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags);
+ DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
OutStoreChains.push_back(Store);
}
SrcOff += VTSize;
@@ -6246,7 +6588,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
uint64_t Size, Align Alignment,
bool isVol, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo) {
+ MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo) {
// Turn a memmove of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -6289,6 +6632,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
}
}
+ // Prepare AAInfo for loads/stores after lowering this memmove.
+ AAMDNodes NewAAInfo = AAInfo;
+ NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
+
MachineMemOperand::Flags MMOFlags =
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
uint64_t SrcOff = 0, DstOff = 0;
@@ -6307,10 +6654,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
if (isDereferenceable)
SrcMMOFlags |= MachineMemOperand::MODereferenceable;
- Value =
- DAG.getLoad(VT, dl, Chain,
- DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
- SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags);
+ Value = DAG.getLoad(
+ VT, dl, Chain,
+ DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
+ SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
LoadValues.push_back(Value);
LoadChains.push_back(Value.getValue(1));
SrcOff += VTSize;
@@ -6322,10 +6669,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
unsigned VTSize = VT.getSizeInBits() / 8;
SDValue Store;
- Store =
- DAG.getStore(Chain, dl, LoadValues[i],
- DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
- DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags);
+ Store = DAG.getStore(
+ Chain, dl, LoadValues[i],
+ DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
+ DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
OutChains.push_back(Store);
DstOff += VTSize;
}
@@ -6354,7 +6701,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
SDValue Chain, SDValue Dst, SDValue Src,
uint64_t Size, Align Alignment, bool isVol,
- MachinePointerInfo DstPtrInfo) {
+ MachinePointerInfo DstPtrInfo,
+ const AAMDNodes &AAInfo) {
// Turn a memset of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -6401,6 +6749,10 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
LargestVT = MemOps[i];
SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
+ // Prepare AAInfo for loads/stores after lowering this memset.
+ AAMDNodes NewAAInfo = AAInfo;
+ NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
+
for (unsigned i = 0; i < NumMemOps; i++) {
EVT VT = MemOps[i];
unsigned VTSize = VT.getSizeInBits() / 8;
@@ -6426,7 +6778,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
Chain, dl, Value,
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
DstPtrInfo.getWithOffset(DstOff), Alignment,
- isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
+ isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
+ NewAAInfo);
OutChains.push_back(Store);
DstOff += VT.getSizeInBits() / 8;
Size -= VTSize;
@@ -6449,7 +6802,8 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool AlwaysInline, bool isTailCall,
MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo) {
+ MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -6460,7 +6814,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Result = getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
- isVol, false, DstPtrInfo, SrcPtrInfo);
+ isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
if (Result.getNode())
return Result;
}
@@ -6481,7 +6835,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
ConstantSize->getZExtValue(), Alignment,
- isVol, true, DstPtrInfo, SrcPtrInfo);
+ isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo);
}
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
@@ -6563,7 +6917,8 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool isTailCall,
MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo) {
+ MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo) {
// Check to see if we should lower the memmove to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -6574,7 +6929,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Result = getMemmoveLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
- isVol, false, DstPtrInfo, SrcPtrInfo);
+ isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
if (Result.getNode())
return Result;
}
@@ -6664,7 +7019,8 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
bool isVol, bool isTailCall,
- MachinePointerInfo DstPtrInfo) {
+ MachinePointerInfo DstPtrInfo,
+ const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -6675,7 +7031,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
ConstantSize->getZExtValue(), Alignment,
- isVol, DstPtrInfo);
+ isVol, DstPtrInfo, AAInfo);
if (Result.getNode())
return Result;
@@ -6839,8 +7195,8 @@ SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
SmallVector<EVT, 4> VTs;
VTs.reserve(Ops.size());
- for (unsigned i = 0; i < Ops.size(); ++i)
- VTs.push_back(Ops[i].getValueType());
+ for (const SDValue &Op : Ops)
+ VTs.push_back(Op.getValueType());
return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
}
@@ -7355,7 +7711,7 @@ SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
AM, ST->isTruncatingStore(), ST->isCompressingStore());
}
-SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
+SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO,
ISD::MemIndexType IndexType,
@@ -7364,9 +7720,9 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
- ID.AddInteger(VT.getRawBits());
+ ID.AddInteger(MemVT.getRawBits());
ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
- dl.getIROrder(), VTs, VT, MMO, IndexType, ExtTy));
+ dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -7374,9 +7730,9 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
return SDValue(E, 0);
}
- IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]);
+ IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
- VTs, VT, MMO, IndexType, ExtTy);
+ VTs, MemVT, MMO, IndexType, ExtTy);
createOperands(N, Ops);
assert(N->getPassThru().getValueType() == N->getValueType(0) &&
@@ -7402,7 +7758,7 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
return V;
}
-SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
+SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO,
ISD::MemIndexType IndexType,
@@ -7411,9 +7767,9 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
- ID.AddInteger(VT.getRawBits());
+ ID.AddInteger(MemVT.getRawBits());
ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
- dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc));
+ dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -7421,9 +7777,9 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
return SDValue(E, 0);
}
- IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]);
+ IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
- VTs, VT, MMO, IndexType, IsTrunc);
+ VTs, MemVT, MMO, IndexType, IsTrunc);
createOperands(N, Ops);
assert(N->getMask().getValueType().getVectorElementCount() ==
@@ -7588,6 +7944,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
default: break;
}
+#ifndef NDEBUG
+ for (auto &Op : Ops)
+ assert(Op.getOpcode() != ISD::DELETED_NODE &&
+ "Operand is DELETED_NODE!");
+#endif
+
switch (Opcode) {
default: break;
case ISD::BUILD_VECTOR:
@@ -7661,6 +8023,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
if (VTList.NumVTs == 1)
return getNode(Opcode, DL, VTList.VTs[0], Ops);
+#ifndef NDEBUG
+ for (auto &Op : Ops)
+ assert(Op.getOpcode() != ISD::DELETED_NODE &&
+ "Operand is DELETED_NODE!");
+#endif
+
switch (Opcode) {
case ISD::STRICT_FP_EXTEND:
assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
@@ -8397,7 +8765,9 @@ SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
"Expected inlined-at fields to agree");
return new (DbgInfo->getAlloc())
- SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
+ SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
+ {}, IsIndirect, DL, O,
+ /*IsVariadic=*/false);
}
/// Constant
@@ -8407,7 +8777,10 @@ SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
const DebugLoc &DL, unsigned O) {
assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
"Expected inlined-at fields to agree");
- return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
+ return new (DbgInfo->getAlloc())
+ SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
+ /*IsIndirect=*/false, DL, O,
+ /*IsVariadic=*/false);
}
/// FrameIndex
@@ -8418,19 +8791,46 @@ SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
unsigned O) {
assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
"Expected inlined-at fields to agree");
+ return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
+}
+
+/// FrameIndex with dependencies
+SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
+ DIExpression *Expr, unsigned FI,
+ ArrayRef<SDNode *> Dependencies,
+ bool IsIndirect,
+ const DebugLoc &DL,
+ unsigned O) {
+ assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
return new (DbgInfo->getAlloc())
- SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX);
+ SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
+ Dependencies, IsIndirect, DL, O,
+ /*IsVariadic=*/false);
}
/// VReg
-SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var,
- DIExpression *Expr,
+SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
unsigned VReg, bool IsIndirect,
const DebugLoc &DL, unsigned O) {
assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
"Expected inlined-at fields to agree");
return new (DbgInfo->getAlloc())
- SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG);
+ SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
+ {}, IsIndirect, DL, O,
+ /*IsVariadic=*/false);
+}
+
+SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
+ ArrayRef<SDDbgOperand> Locs,
+ ArrayRef<SDNode *> Dependencies,
+ bool IsIndirect, const DebugLoc &DL,
+ unsigned O, bool IsVariadic) {
+ assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
+ "Expected inlined-at fields to agree");
+ return new (DbgInfo->getAlloc())
+ SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
+ DL, O, IsVariadic);
}
void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
@@ -8449,15 +8849,31 @@ void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
if (!FromNode->getHasDebugValue())
return;
+ SDDbgOperand FromLocOp =
+ SDDbgOperand::fromNode(From.getNode(), From.getResNo());
+ SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());
+
SmallVector<SDDbgValue *, 2> ClonedDVs;
for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
- if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
+ if (Dbg->isInvalidated())
continue;
// TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
- // Just transfer the dbg value attached to From.
- if (Dbg->getResNo() != From.getResNo())
+ // Create a new location ops vector that is equal to the old vector, but
+ // with each instance of FromLocOp replaced with ToLocOp.
+ bool Changed = false;
+ auto NewLocOps = Dbg->copyLocationOps();
+ std::replace_if(
+ NewLocOps.begin(), NewLocOps.end(),
+ [&Changed, FromLocOp](const SDDbgOperand &Op) {
+ bool Match = Op == FromLocOp;
+ Changed |= Match;
+ return Match;
+ },
+ ToLocOp);
+ // Ignore this SDDbgValue if we didn't find a matching location.
+ if (!Changed)
continue;
DIVariable *Var = Dbg->getVariable();
@@ -8476,10 +8892,13 @@ void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
continue;
Expr = *Fragment;
}
+
+ auto AdditionalDependencies = Dbg->getAdditionalDependencies();
// Clone the SDDbgValue and move it to To.
- SDDbgValue *Clone = getDbgValue(
- Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), Dbg->getDebugLoc(),
- std::max(ToNode->getIROrder(), Dbg->getOrder()));
+ SDDbgValue *Clone = getDbgValueList(
+ Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
+ Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
+ Dbg->isVariadic());
ClonedDVs.push_back(Clone);
if (InvalidateDbg) {
@@ -8489,8 +8908,11 @@ void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
}
}
- for (SDDbgValue *Dbg : ClonedDVs)
- AddDbgValue(Dbg, ToNode, false);
+ for (SDDbgValue *Dbg : ClonedDVs) {
+ assert(is_contained(Dbg->getSDNodes(), ToNode) &&
+ "Transferred DbgValues should depend on the new SDNode");
+ AddDbgValue(Dbg, false);
+ }
}
void SelectionDAG::salvageDebugInfo(SDNode &N) {
@@ -8510,16 +8932,35 @@ void SelectionDAG::salvageDebugInfo(SDNode &N) {
if (!isConstantIntBuildVectorOrConstantInt(N0) &&
isConstantIntBuildVectorOrConstantInt(N1)) {
uint64_t Offset = N.getConstantOperandVal(1);
+
// Rewrite an ADD constant node into a DIExpression. Since we are
// performing arithmetic to compute the variable's *value* in the
// DIExpression, we need to mark the expression with a
// DW_OP_stack_value.
auto *DIExpr = DV->getExpression();
- DIExpr =
- DIExpression::prepend(DIExpr, DIExpression::StackValue, Offset);
- SDDbgValue *Clone =
- getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(),
- DV->isIndirect(), DV->getDebugLoc(), DV->getOrder());
+ auto NewLocOps = DV->copyLocationOps();
+ bool Changed = false;
+ for (size_t i = 0; i < NewLocOps.size(); ++i) {
+ // We're not given a ResNo to compare against because the whole
+ // node is going away. We know that any ISD::ADD only has one
+ // result, so we can assume any node match is using the result.
+ if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
+ NewLocOps[i].getSDNode() != &N)
+ continue;
+ NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
+ SmallVector<uint64_t, 3> ExprOps;
+ DIExpression::appendOffset(ExprOps, Offset);
+ DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
+ Changed = true;
+ }
+ (void)Changed;
+ assert(Changed && "Salvage target doesn't use N");
+
+ auto AdditionalDependencies = DV->getAdditionalDependencies();
+ SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr,
+ NewLocOps, AdditionalDependencies,
+ DV->isIndirect(), DV->getDebugLoc(),
+ DV->getOrder(), DV->isVariadic());
ClonedDVs.push_back(Clone);
DV->setIsInvalidated();
DV->setIsEmitted();
@@ -8530,8 +8971,11 @@ void SelectionDAG::salvageDebugInfo(SDNode &N) {
}
}
- for (SDDbgValue *Dbg : ClonedDVs)
- AddDbgValue(Dbg, Dbg->getSDNode(), false);
+ for (SDDbgValue *Dbg : ClonedDVs) {
+ assert(!Dbg->getSDNodes().empty() &&
+ "Salvaged DbgValue should depend on a new SDNode");
+ AddDbgValue(Dbg, false);
+ }
}
/// Creates a SDDbgLabel node.
@@ -8965,9 +9409,7 @@ unsigned SelectionDAG::AssignTopologicalOrder() {
checkForCycles(N, this);
// N is in sorted position, so all its uses have one less operand
// that needs to be sorted.
- for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
- UI != UE; ++UI) {
- SDNode *P = *UI;
+ for (SDNode *P : N->uses()) {
unsigned Degree = P->getNodeId();
assert(Degree != 0 && "Invalid node degree");
--Degree;
@@ -9014,17 +9456,17 @@ unsigned SelectionDAG::AssignTopologicalOrder() {
/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
/// value is produced by SD.
-void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
- if (SD) {
+void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
+ for (SDNode *SD : DB->getSDNodes()) {
+ if (!SD)
+ continue;
assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
SD->setHasDebugValue(true);
}
- DbgInfo->add(DB, SD, isParameter);
+ DbgInfo->add(DB, isParameter);
}
-void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) {
- DbgInfo->add(DB);
-}
+void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
SDValue NewMemOpChain) {
@@ -9226,21 +9668,22 @@ ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
// TODO: may want to use peekThroughBitcast() here.
- ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
+ ConstantSDNode *C =
+ isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
return C && C->isNullValue();
}
-bool llvm::isOneOrOneSplat(SDValue N) {
+bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
// TODO: may want to use peekThroughBitcast() here.
unsigned BitWidth = N.getScalarValueSizeInBits();
- ConstantSDNode *C = isConstOrConstSplat(N);
+ ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;
}
-bool llvm::isAllOnesOrAllOnesSplat(SDValue N) {
+bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
N = peekThroughBitcasts(N);
unsigned BitWidth = N.getScalarValueSizeInBits();
- ConstantSDNode *C = isConstOrConstSplat(N);
+ ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth;
}
@@ -9290,8 +9733,8 @@ namespace {
std::vector<EVT> VTs;
EVTArray() {
- VTs.reserve(MVT::LAST_VALUETYPE);
- for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
+ VTs.reserve(MVT::VALUETYPE_SIZE);
+ for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
VTs.push_back(MVT((MVT::SimpleValueType)i));
}
};
@@ -9308,11 +9751,9 @@ const EVT *SDNode::getValueTypeList(EVT VT) {
if (VT.isExtended()) {
sys::SmartScopedLock<true> Lock(*VTMutex);
return &(*EVTs->insert(VT).first);
- } else {
- assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
- "Value type out of range!");
- return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
}
+ assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
+ return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
}
/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
@@ -9890,10 +10331,10 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
// FIXME: This does not work for vectors with elements less than 8 bits.
while (VecWidth > 8) {
unsigned HalfSize = VecWidth / 2;
- APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
- APInt LowValue = SplatValue.trunc(HalfSize);
- APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
- APInt LowUndef = SplatUndef.trunc(HalfSize);
+ APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
+ APInt LowValue = SplatValue.extractBits(HalfSize, 0);
+ APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
+ APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
// If the two halves do not match (ignoring undef bits), stop here.
if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
generated by cgit v1.3 (git 2.53.0) at 2026-09-03 09:42:21 +0000