summaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/SelectionDAG
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp655
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp8
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp16
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp10
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp99
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp7
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp30
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp8
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp61
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp82
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h5
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp20
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp59
13 files changed, 600 insertions, 460 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index df5a041b87cd..067ad819e0d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -487,10 +487,7 @@ namespace {
SDValue visitFCEIL(SDNode *N);
SDValue visitFTRUNC(SDNode *N);
SDValue visitFFLOOR(SDNode *N);
- SDValue visitFMINNUM(SDNode *N);
- SDValue visitFMAXNUM(SDNode *N);
- SDValue visitFMINIMUM(SDNode *N);
- SDValue visitFMAXIMUM(SDNode *N);
+ SDValue visitFMinMax(SDNode *N);
SDValue visitBRCOND(SDNode *N);
SDValue visitBR_CC(SDNode *N);
SDValue visitLOAD(SDNode *N);
@@ -1701,10 +1698,10 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::FNEG: return visitFNEG(N);
case ISD::FABS: return visitFABS(N);
case ISD::FFLOOR: return visitFFLOOR(N);
- case ISD::FMINNUM: return visitFMINNUM(N);
- case ISD::FMAXNUM: return visitFMAXNUM(N);
- case ISD::FMINIMUM: return visitFMINIMUM(N);
- case ISD::FMAXIMUM: return visitFMAXIMUM(N);
+ case ISD::FMINNUM:
+ case ISD::FMAXNUM:
+ case ISD::FMINIMUM:
+ case ISD::FMAXIMUM: return visitFMinMax(N);
case ISD::FCEIL: return visitFCEIL(N);
case ISD::FTRUNC: return visitFTRUNC(N);
case ISD::BRCOND: return visitBRCOND(N);
@@ -2260,6 +2257,21 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
EVT VT = N0.getValueType();
SDLoc DL(N);
+ // fold (add x, undef) -> undef
+ if (N0.isUndef())
+ return N0;
+ if (N1.isUndef())
+ return N1;
+
+ // fold (add c1, c2) -> c1+c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
+
// fold vector ops
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
@@ -2268,23 +2280,6 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
// fold (add x, 0) -> x, vector edition
if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
return N0;
- if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
- return N1;
- }
-
- // fold (add x, undef) -> undef
- if (N0.isUndef())
- return N0;
-
- if (N1.isUndef())
- return N1;
-
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
- // canonicalize constant to RHS
- if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
- // fold (add c1, c2) -> c1+c2
- return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1});
}
// fold (add x, 0) -> x
@@ -2554,6 +2549,19 @@ SDValue DAGCombiner::visitADDSAT(SDNode *N) {
EVT VT = N0.getValueType();
SDLoc DL(N);
+ // fold (add_sat x, undef) -> -1
+ if (N0.isUndef() || N1.isUndef())
+ return DAG.getAllOnesConstant(DL, VT);
+
+ // fold (add_sat c1, c2) -> c3
+ if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(Opcode, DL, VT, N1, N0);
+
// fold vector ops
if (VT.isVector()) {
// TODO SimplifyVBinOp
@@ -2561,20 +2569,6 @@ SDValue DAGCombiner::visitADDSAT(SDNode *N) {
// fold (add_sat x, 0) -> x, vector edition
if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
return N0;
- if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
- return N1;
- }
-
- // fold (add_sat x, undef) -> -1
- if (N0.isUndef() || N1.isUndef())
- return DAG.getAllOnesConstant(DL, VT);
-
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
- // canonicalize constant to RHS
- if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(Opcode, DL, VT, N1, N0);
- // fold (add_sat c1, c2) -> c3
- return DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1});
}
// fold (add_sat x, 0) -> x
@@ -3260,6 +3254,15 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
EVT VT = N0.getValueType();
SDLoc DL(N);
+ // fold (sub x, x) -> 0
+ // FIXME: Refactor this and xor and other similar operations together.
+ if (N0 == N1)
+ return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
+
+ // fold (sub c1, c2) -> c3
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
@@ -3270,15 +3273,6 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
return N0;
}
- // fold (sub x, x) -> 0
- // FIXME: Refactor this and xor and other similar operations together.
- if (N0 == N1)
- return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
-
- // fold (sub c1, c2) -> c3
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
- return C;
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -3611,15 +3605,6 @@ SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
EVT VT = N0.getValueType();
SDLoc DL(N);
- // fold vector ops
- if (VT.isVector()) {
- // TODO SimplifyVBinOp
-
- // fold (sub_sat x, 0) -> x, vector edition
- if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
- return N0;
- }
-
// fold (sub_sat x, undef) -> 0
if (N0.isUndef() || N1.isUndef())
return DAG.getConstant(0, DL, VT);
@@ -3632,6 +3617,15 @@ SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
return C;
+ // fold vector ops
+ if (VT.isVector()) {
+ // TODO SimplifyVBinOp
+
+ // fold (sub_sat x, 0) -> x, vector edition
+ if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
+ return N0;
+ }
+
// fold (sub_sat x, 0) -> x
if (isNullConstant(N1))
return N0;
@@ -3781,6 +3775,15 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
if (N0.isUndef() || N1.isUndef())
return DAG.getConstant(0, SDLoc(N), VT);
+ // fold (mul c1, c2) -> c1*c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS (vector doesn't have to splat)
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
+
bool N1IsConst = false;
bool N1IsOpaqueConst = false;
APInt ConstValue1;
@@ -3802,15 +3805,6 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
}
}
- // fold (mul c1, c2) -> c1*c2
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
- return C;
-
- // canonicalize constant to RHS (vector doesn't have to splat)
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
-
// fold (mul x, 0) -> 0
if (N1IsConst && ConstValue1.isZero())
return N1;
@@ -4140,17 +4134,17 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
EVT CCVT = getSetCCResultType(VT);
SDLoc DL(N);
+ // fold (sdiv c1, c2) -> c1/c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector())
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
- // fold (sdiv c1, c2) -> c1/c2
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
- return C;
-
// fold (sdiv X, -1) -> 0-X
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C && N1C->isAllOnes())
return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
@@ -4284,17 +4278,17 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) {
EVT CCVT = getSetCCResultType(VT);
SDLoc DL(N);
+ // fold (udiv c1, c2) -> c1/c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector())
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
- // fold (udiv c1, c2) -> c1/c2
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
- return C;
-
// fold (udiv X, -1) -> select(X == -1, 1, 0)
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C && N1C->isAllOnes())
return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
DAG.getConstant(1, DL, VT),
@@ -4463,6 +4457,15 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
EVT VT = N->getValueType(0);
SDLoc DL(N);
+ // fold (mulhs c1, c2)
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
+
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
@@ -4474,15 +4477,6 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
return DAG.getConstant(0, DL, VT);
}
- // fold (mulhs c1, c2)
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
- return C;
-
- // canonicalize constant to RHS.
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
-
// fold (mulhs x, 0) -> 0
if (isNullConstant(N1))
return N1;
@@ -4523,6 +4517,15 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
EVT VT = N->getValueType(0);
SDLoc DL(N);
+ // fold (mulhu c1, c2)
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
+
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
@@ -4534,15 +4537,6 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
return DAG.getConstant(0, DL, VT);
}
- // fold (mulhu c1, c2)
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
- return C;
-
- // canonicalize constant to RHS.
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
-
// fold (mulhu x, 0) -> 0
if (isNullConstant(N1))
return N1;
@@ -4786,12 +4780,14 @@ SDValue DAGCombiner::visitMULO(SDNode *N) {
}
// Function to calculate whether the Min/Max pair of SDNodes (potentially
-// swapped around) make a signed saturate pattern, clamping to between -2^(BW-1)
-// and 2^(BW-1)-1. Returns the node being clamped and the bitwidth of the clamp
-// in BW. Should work with both SMIN/SMAX nodes and setcc/select combo. The
-// operands are the same as SimplifySelectCC. N0<N1 ? N2 : N3
+// swapped around) make a signed saturate pattern, clamping to between a signed
+// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
+// Returns the node being clamped and the bitwidth of the clamp in BW. Should
+// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
+// same as SimplifySelectCC. N0<N1 ? N2 : N3.
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
- SDValue N3, ISD::CondCode CC, unsigned &BW) {
+ SDValue N3, ISD::CondCode CC, unsigned &BW,
+ bool &Unsigned) {
auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
ISD::CondCode CC) {
// The compare and select operand should be the same or the select operands
@@ -4858,17 +4854,27 @@ static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
const APInt &MinC = MinCOp->getAPIntValue();
const APInt &MaxC = MaxCOp->getAPIntValue();
APInt MinCPlus1 = MinC + 1;
- if (-MaxC != MinCPlus1 || !MinCPlus1.isPowerOf2())
- return SDValue();
- BW = MinCPlus1.exactLogBase2() + 1;
- return N02;
+ if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
+ BW = MinCPlus1.exactLogBase2() + 1;
+ Unsigned = false;
+ return N02;
+ }
+
+ if (MaxC == 0 && MinCPlus1.isPowerOf2()) {
+ BW = MinCPlus1.exactLogBase2();
+ Unsigned = true;
+ return N02;
+ }
+
+ return SDValue();
}
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
SDValue N3, ISD::CondCode CC,
SelectionDAG &DAG) {
unsigned BW;
- SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW);
+ bool Unsigned;
+ SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned);
if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
return SDValue();
EVT FPVT = Fp.getOperand(0).getValueType();
@@ -4876,13 +4882,14 @@ static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
if (FPVT.isVector())
NewVT = EVT::getVectorVT(*DAG.getContext(), NewVT,
FPVT.getVectorElementCount());
- if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(
- ISD::FP_TO_SINT_SAT, Fp.getOperand(0).getValueType(), NewVT))
+ unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
+ if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(NewOpc, FPVT, NewVT))
return SDValue();
SDLoc DL(Fp);
- SDValue Sat = DAG.getNode(ISD::FP_TO_SINT_SAT, DL, NewVT, Fp.getOperand(0),
+ SDValue Sat = DAG.getNode(NewOpc, DL, NewVT, Fp.getOperand(0),
DAG.getValueType(NewVT.getScalarType()));
- return DAG.getSExtOrTrunc(Sat, DL, N2->getValueType(0));
+ return Unsigned ? DAG.getZExtOrTrunc(Sat, DL, N2->getValueType(0))
+ : DAG.getSExtOrTrunc(Sat, DL, N2->getValueType(0));
}
SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
@@ -4892,11 +4899,6 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
unsigned Opcode = N->getOpcode();
SDLoc DL(N);
- // fold vector ops
- if (VT.isVector())
- if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
- return FoldedVOp;
-
// fold operation with constant operands.
if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
return C;
@@ -4904,7 +4906,12 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
// canonicalize constant to RHS
if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
!DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(N->getOpcode(), DL, VT, N1, N0);
+ return DAG.getNode(Opcode, DL, VT, N1, N0);
+
+ // fold vector ops
+ if (VT.isVector())
+ if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+ return FoldedVOp;
// Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
// Only do this if the current op isn't legal and the flipped is.
@@ -5777,6 +5784,15 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
if (N0 == N1)
return N0;
+ // fold (and c1, c2) -> c1&c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
+
// fold vector ops
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
@@ -5824,22 +5840,13 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
}
}
- // fold (and c1, c2) -> c1&c2
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
- return C;
-
- // canonicalize constant to RHS
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
-
// fold (and x, -1) -> x
if (isAllOnesConstant(N1))
return N0;
// if (and x, c) is known to be zero, return 0
unsigned BitWidth = VT.getScalarSizeInBits();
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(BitWidth)))
return DAG.getConstant(0, SDLoc(N), VT);
@@ -6546,21 +6553,25 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
if (N0 == N1)
return N0;
+ // fold (or c1, c2) -> c1|c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1}))
+ return C;
+
+ // canonicalize constant to RHS
+ if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
+
// fold vector ops
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
return FoldedVOp;
// fold (or x, 0) -> x, vector edition
- if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
- return N1;
if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
return N0;
// fold (or x, -1) -> -1, vector edition
- if (ISD::isConstantSplatVectorAllOnes(N0.getNode()))
- // do not return N0, because undef node may exist in N0
- return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
if (ISD::isConstantSplatVectorAllOnes(N1.getNode()))
// do not return N1, because undef node may exist in N1
return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
@@ -6629,16 +6640,6 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
}
}
- // fold (or c1, c2) -> c1|c2
- ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1}))
- return C;
-
- // canonicalize constant to RHS
- if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
- return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
-
// fold (or x, 0) -> x
if (isNullConstant(N1))
return N0;
@@ -6651,6 +6652,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
return NewSel;
// fold (or x, c) -> c iff (x & ~c) == 0
+ ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
return N1;
@@ -7941,18 +7943,6 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
EVT VT = N0.getValueType();
SDLoc DL(N);
- // fold vector ops
- if (VT.isVector()) {
- if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
- return FoldedVOp;
-
- // fold (xor x, 0) -> x, vector edition
- if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
- return N1;
- if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
- return N0;
- }
-
// fold (xor undef, undef) -> 0. This is a common idiom (misuse).
if (N0.isUndef() && N1.isUndef())
return DAG.getConstant(0, DL, VT);
@@ -7969,9 +7959,19 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
// canonicalize constant to RHS
if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
- !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+ !DAG.isConstantIntBuildVectorOrConstantInt(N1))
return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
+ // fold vector ops
+ if (VT.isVector()) {
+ if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+ return FoldedVOp;
+
+ // fold (xor x, 0) -> x, vector edition
+ if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
+ return N0;
+ }
+
// fold (xor x, 0) -> x
if (isNullConstant(N1))
return N0;
@@ -8409,6 +8409,10 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
EVT ShiftVT = N1.getValueType();
unsigned OpSizeInBits = VT.getScalarSizeInBits();
+ // fold (shl c1, c2) -> c1<<c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector()) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
@@ -8434,12 +8438,6 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
}
}
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
- // fold (shl c1, c2) -> c1<<c2
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1}))
- return C;
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -8558,6 +8556,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
// fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
// fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2
// TODO - support non-uniform vector shift amounts.
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
N0->getFlags().hasExact()) {
if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
@@ -8758,6 +8757,10 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
EVT VT = N0.getValueType();
unsigned OpSizeInBits = VT.getScalarSizeInBits();
+ // fold (sra c1, c2) -> (sra c1, c2)
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1}))
+ return C;
+
// Arithmetic shifting an all-sign-bit value is a no-op.
// fold (sra 0, x) -> 0
// fold (sra -1, x) -> -1
@@ -8769,17 +8772,12 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
return FoldedVOp;
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
- // fold (sra c1, c2) -> (sra c1, c2)
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1}))
- return C;
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
// fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
// sext_inreg.
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
@@ -8962,21 +8960,20 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
EVT VT = N0.getValueType();
unsigned OpSizeInBits = VT.getScalarSizeInBits();
+ // fold (srl c1, c2) -> c1 >>u c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector())
if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
return FoldedVOp;
- ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
- // fold (srl c1, c2) -> c1 >>u c2
- if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1}))
- return C;
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
// if (srl x, c) is known to be zero, return 0
+ ConstantSDNode *N1C = isConstOrConstSplat(N1);
if (N1C &&
DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
return DAG.getConstant(0, SDLoc(N), VT);
@@ -10043,6 +10040,8 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
SDValue Mask = MST->getMask();
SDValue Chain = MST->getChain();
+ SDValue Value = MST->getValue();
+ SDValue Ptr = MST->getBasePtr();
SDLoc DL(N);
// Zap masked stores with a zero mask.
@@ -10054,12 +10053,50 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() &&
!MST->isCompressingStore() && !MST->isTruncatingStore())
return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
- MST->getBasePtr(), MST->getMemOperand());
+ MST->getBasePtr(), MST->getPointerInfo(),
+ MST->getOriginalAlign(), MachineMemOperand::MOStore,
+ MST->getAAInfo());
// Try transforming N to an indexed store.
if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
return SDValue(N, 0);
+ if (MST->isTruncatingStore() && MST->isUnindexed() &&
+ Value.getValueType().isInteger() &&
+ (!isa<ConstantSDNode>(Value) ||
+ !cast<ConstantSDNode>(Value)->isOpaque())) {
+ APInt TruncDemandedBits =
+ APInt::getLowBitsSet(Value.getScalarValueSizeInBits(),
+ MST->getMemoryVT().getScalarSizeInBits());
+
+ // See if we can simplify the operation with
+ // SimplifyDemandedBits, which only works if the value has a single use.
+ if (SimplifyDemandedBits(Value, TruncDemandedBits)) {
+ // Re-visit the store if anything changed and the store hasn't been merged
+ // with another node (N is deleted) SimplifyDemandedBits will add Value's
+ // node back to the worklist if necessary, but we also need to re-visit
+ // the Store node itself.
+ if (N->getOpcode() != ISD::DELETED_NODE)
+ AddToWorklist(N);
+ return SDValue(N, 0);
+ }
+ }
+
+ // If this is a TRUNC followed by a masked store, fold this into a masked
+ // truncating store. We can do this even if this is already a masked
+ // truncstore.
+ if ((Value.getOpcode() == ISD::TRUNCATE) && Value.getNode()->hasOneUse() &&
+ MST->isUnindexed() &&
+ TLI.canCombineTruncStore(Value.getOperand(0).getValueType(),
+ MST->getMemoryVT(), LegalOperations)) {
+ auto Mask = TLI.promoteTargetBoolean(DAG, MST->getMask(),
+ Value.getOperand(0).getValueType());
+ return DAG.getMaskedStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
+ MST->getOffset(), Mask, MST->getMemoryVT(),
+ MST->getMemOperand(), MST->getAddressingMode(),
+ /*IsTruncating=*/true);
+ }
+
return SDValue();
}
@@ -10109,8 +10146,10 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
// FIXME: Can we do this for indexed, expanding, or extending loads?
if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() &&
!MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
- SDValue NewLd = DAG.getLoad(N->getValueType(0), SDLoc(N), MLD->getChain(),
- MLD->getBasePtr(), MLD->getMemOperand());
+ SDValue NewLd = DAG.getLoad(
+ N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(),
+ MLD->getPointerInfo(), MLD->getOriginalAlign(),
+ MachineMemOperand::MOLoad, MLD->getAAInfo(), MLD->getRanges());
return CombineTo(N, NewLd, NewLd.getValue(1));
}
@@ -13876,19 +13915,19 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
return R;
- // fold vector ops
- if (VT.isVector())
- if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
- return FoldedVOp;
-
// fold (fadd c1, c2) -> c1 + c2
- if (N0CFP && N1CFP)
- return DAG.getNode(ISD::FADD, DL, VT, N0, N1);
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1}))
+ return C;
// canonicalize constant to RHS
if (N0CFP && !N1CFP)
return DAG.getNode(ISD::FADD, DL, VT, N1, N0);
+ // fold vector ops
+ if (VT.isVector())
+ if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+ return FoldedVOp;
+
// N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math)
ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, true);
if (N1C && N1C->isZero())
@@ -14084,15 +14123,15 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
return R;
+ // fold (fsub c1, c2) -> c1-c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector())
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
- // fold (fsub c1, c2) -> c1-c2
- if (N0CFP && N1CFP)
- return DAG.getNode(ISD::FSUB, DL, VT, N0, N1);
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -14157,7 +14196,6 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
SDValue DAGCombiner::visitFMUL(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
- ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, true);
ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true);
EVT VT = N->getValueType(0);
SDLoc DL(N);
@@ -14168,22 +14206,20 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
return R;
- // fold vector ops
- if (VT.isVector()) {
- // This just handles C1 * C2 for vectors. Other vector folds are below.
- if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
- return FoldedVOp;
- }
-
// fold (fmul c1, c2) -> c1*c2
- if (N0CFP && N1CFP)
- return DAG.getNode(ISD::FMUL, DL, VT, N0, N1);
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1}))
+ return C;
// canonicalize constant to RHS
if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
!DAG.isConstantFPBuildVectorOrConstantFP(N1))
return DAG.getNode(ISD::FMUL, DL, VT, N1, N0);
+ // fold vector ops
+ if (VT.isVector())
+ if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+ return FoldedVOp;
+
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -14495,8 +14531,6 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
SDValue DAGCombiner::visitFDIV(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
- ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
- ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
EVT VT = N->getValueType(0);
SDLoc DL(N);
const TargetOptions &Options = DAG.getTarget().Options;
@@ -14506,15 +14540,15 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
return R;
+ // fold (fdiv c1, c2) -> c1/c2
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1}))
+ return C;
+
// fold vector ops
if (VT.isVector())
if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
return FoldedVOp;
- // fold (fdiv c1, c2) -> c1/c2
- if (N0CFP && N1CFP)
- return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1);
-
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -14523,7 +14557,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
// fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
- if (N1CFP) {
+ if (auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
// Compute the reciprocal 1.0 / c2.
const APFloat &N1APF = N1CFP->getValueAPF();
APFloat Recip(N1APF.getSemantics(), 1); // 1.0
@@ -14639,8 +14673,6 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
SDValue DAGCombiner::visitFREM(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
- ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
- ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
EVT VT = N->getValueType(0);
SDNodeFlags Flags = N->getFlags();
SelectionDAG::FlagInserter FlagsInserter(DAG, N);
@@ -14649,9 +14681,9 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
return R;
// fold (frem c1, c2) -> fmod(c1,c2)
- if (N0CFP && N1CFP)
- return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1);
-
+ if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1}))
+ return C;
+
if (SDValue NewSel = foldBinOpIntoSelect(N))
return NewSel;
@@ -14712,12 +14744,12 @@ static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N) {
SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
- bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
- bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
EVT VT = N->getValueType(0);
- if (N0CFP && N1CFP) // Constant fold
- return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1);
+ // fold (fcopysign c1, c2) -> fcopysign(c1,c2)
+ if (SDValue C =
+ DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1}))
+ return C;
if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N->getOperand(1))) {
const APFloat &V = N1C->getValueAPF();
@@ -14835,14 +14867,6 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) {
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI) {
- // This optimization is guarded by a function attribute because it may produce
- // unexpected results. Ie, programs may be relying on the platform-specific
- // undefined behavior when the float-to-int conversion overflows.
- const Function &F = DAG.getMachineFunction().getFunction();
- Attribute StrictOverflow = F.getFnAttribute("strict-float-cast-overflow");
- if (StrictOverflow.getValueAsString().equals("false"))
- return SDValue();
-
// We only do this if the target has legal ftrunc. Otherwise, we'd likely be
// replacing casts with a libcall. We also must be allowed to ignore -0.0
// because FTRUNC will return -0.0 for (-1.0, -0.0), but using integer
@@ -15216,31 +15240,26 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
return SDValue();
}
-static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
- APFloat (*Op)(const APFloat &, const APFloat &)) {
+SDValue DAGCombiner::visitFMinMax(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
EVT VT = N->getValueType(0);
- const ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0);
- const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1);
const SDNodeFlags Flags = N->getFlags();
unsigned Opc = N->getOpcode();
bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
SelectionDAG::FlagInserter FlagsInserter(DAG, N);
- if (N0CFP && N1CFP) {
- const APFloat &C0 = N0CFP->getValueAPF();
- const APFloat &C1 = N1CFP->getValueAPF();
- return DAG.getConstantFP(Op(C0, C1), SDLoc(N), VT);
- }
+ // Constant fold.
+ if (SDValue C = DAG.FoldConstantArithmetic(Opc, SDLoc(N), VT, {N0, N1}))
+ return C;
// Canonicalize to constant on RHS.
if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
!DAG.isConstantFPBuildVectorOrConstantFP(N1))
return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
- if (N1CFP) {
+ if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) {
const APFloat &AF = N1CFP->getValueAPF();
// minnum(X, nan) -> X
@@ -15272,22 +15291,6 @@ static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
return SDValue();
}
-SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
- return visitFMinMax(DAG, N, minnum);
-}
-
-SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
- return visitFMinMax(DAG, N, maxnum);
-}
-
-SDValue DAGCombiner::visitFMINIMUM(SDNode *N) {
- return visitFMinMax(DAG, N, minimum);
-}
-
-SDValue DAGCombiner::visitFMAXIMUM(SDNode *N) {
- return visitFMinMax(DAG, N, maximum);
-}
-
SDValue DAGCombiner::visitFABS(SDNode *N) {
SDValue N0 = N->getOperand(0);
EVT VT = N->getValueType(0);
@@ -18392,8 +18395,8 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
if (ST->isUnindexed() && ST->isSimple() &&
ST1->isUnindexed() && ST1->isSimple()) {
- if (ST1->getBasePtr() == Ptr && ST1->getValue() == Value &&
- ST->getMemoryVT() == ST1->getMemoryVT() &&
+ if (OptLevel != CodeGenOpt::None && ST1->getBasePtr() == Ptr &&
+ ST1->getValue() == Value && ST->getMemoryVT() == ST1->getMemoryVT() &&
ST->getAddressSpace() == ST1->getAddressSpace()) {
// If this is a store followed by a store with the same value to the
// same location, then the store is dead/noop.
@@ -20727,6 +20730,156 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
return NewLd;
}
+/// Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)),
+/// try to produce VECTOR_SHUFFLE(EXTRACT_SUBVECTOR(Op?, ?),
+/// EXTRACT_SUBVECTOR(Op?, ?),
+/// Mask'))
+/// iff it is legal and profitable to do so. Notably, the trimmed mask
+/// (containing only the elements that are extracted)
+/// must reference at most two subvectors.
+static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
+ SelectionDAG &DAG,
+ const TargetLowering &TLI,
+ bool LegalOperations) {
+ assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+ "Must only be called on EXTRACT_SUBVECTOR's");
+
+ SDValue N0 = N->getOperand(0);
+
+ // Only deal with non-scalable vectors.
+ EVT NarrowVT = N->getValueType(0);
+ EVT WideVT = N0.getValueType();
+ if (!NarrowVT.isFixedLengthVector() || !WideVT.isFixedLengthVector())
+ return SDValue();
+
+ // The operand must be a shufflevector.
+ auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0);
+ if (!WideShuffleVector)
+ return SDValue();
+
+ // The old shuffleneeds to go away.
+ if (!WideShuffleVector->hasOneUse())
+ return SDValue();
+
+ // And the narrow shufflevector that we'll form must be legal.
+ if (LegalOperations &&
+ !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, NarrowVT))
+ return SDValue();
+
+ uint64_t FirstExtractedEltIdx = N->getConstantOperandVal(1);
+ int NumEltsExtracted = NarrowVT.getVectorNumElements();
+ assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
+ "Extract index is not a multiple of the output vector length.");
+
+ int WideNumElts = WideVT.getVectorNumElements();
+
+ SmallVector<int, 16> NewMask;
+ NewMask.reserve(NumEltsExtracted);
+ SmallSetVector<std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>, 2>
+ DemandedSubvectors;
+
+ // Try to decode the wide mask into narrow mask from at most two subvectors.
+ for (int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
+ NumEltsExtracted)) {
+ assert((M >= -1) && (M < (2 * WideNumElts)) &&
+ "Out-of-bounds shuffle mask?");
+
+ if (M < 0) {
+ // Does not depend on operands, does not require adjustment.
+ NewMask.emplace_back(M);
+ continue;
+ }
+
+ // From which operand of the shuffle does this shuffle mask element pick?
+ int WideShufOpIdx = M / WideNumElts;
+ // Which element of that operand is picked?
+ int OpEltIdx = M % WideNumElts;
+
+ assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
+ "Shuffle mask vector decomposition failure.");
+
+ // And which NumEltsExtracted-sized subvector of that operand is that?
+ int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
+ // And which element within that subvector of that operand is that?
+ int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
+
+ assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
+ "Shuffle mask subvector decomposition failure.");
+
+ assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
+ WideShufOpIdx * WideNumElts) == M &&
+ "Shuffle mask full decomposition failure.");
+
+ SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
+
+ if (Op.isUndef()) {
+ // Picking from an undef operand. Let's adjust mask instead.
+ NewMask.emplace_back(-1);
+ continue;
+ }
+
+ // Profitability check: only deal with extractions from the first subvector.
+ if (OpSubvecIdx != 0)
+ return SDValue();
+
+ const std::pair<SDValue, int> DemandedSubvector =
+ std::make_pair(Op, OpSubvecIdx);
+
+ if (DemandedSubvectors.insert(DemandedSubvector)) {
+ if (DemandedSubvectors.size() > 2)
+ return SDValue(); // We can't handle more than two subvectors.
+ // How many elements into the WideVT does this subvector start?
+ int Index = NumEltsExtracted * OpSubvecIdx;
+ // Bail out if the extraction isn't going to be cheap.
+ if (!TLI.isExtractSubvectorCheap(NarrowVT, WideVT, Index))
+ return SDValue();
+ }
+
+ // Ok, but from which operand of the new shuffle will this element pick?
+ int NewOpIdx =
+ getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
+ assert((NewOpIdx == 0 || NewOpIdx == 1) && "Unexpected operand index.");
+
+ int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
+ NewMask.emplace_back(AdjM);
+ }
+ assert(NewMask.size() == (unsigned)NumEltsExtracted && "Produced bad mask.");
+ assert(DemandedSubvectors.size() <= 2 &&
+ "Should have ended up demanding at most two subvectors.");
+
+ // Did we discover that the shuffle does not actually depend on operands?
+ if (DemandedSubvectors.empty())
+ return DAG.getUNDEF(NarrowVT);
+
+ // We still perform the exact same EXTRACT_SUBVECTOR, just on different
+ // operand[s]/index[es], so there is no point in checking for it's legality.
+
+ // Do not turn a legal shuffle into an illegal one.
+ if (TLI.isShuffleMaskLegal(WideShuffleVector->getMask(), WideVT) &&
+ !TLI.isShuffleMaskLegal(NewMask, NarrowVT))
+ return SDValue();
+
+ SDLoc DL(N);
+
+ SmallVector<SDValue, 2> NewOps;
+ for (const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
+ &DemandedSubvector : DemandedSubvectors) {
+ // How many elements into the WideVT does this subvector start?
+ int Index = NumEltsExtracted * DemandedSubvector.second;
+ SDValue IndexC = DAG.getVectorIdxConstant(Index, DL);
+ NewOps.emplace_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT,
+ DemandedSubvector.first, IndexC));
+ }
+ assert((NewOps.size() == 1 || NewOps.size() == 2) &&
+ "Should end up with either one or two ops");
+
+ // If we ended up with only one operand, pad with an undef.
+ if (NewOps.size() == 1)
+ NewOps.emplace_back(DAG.getUNDEF(NarrowVT));
+
+ return DAG.getVectorShuffle(NarrowVT, DL, NewOps[0], NewOps[1], NewMask);
+}
+
SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
EVT NVT = N->getValueType(0);
SDValue V = N->getOperand(0);
@@ -20840,6 +20993,10 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
}
}
+ if (SDValue V =
+ foldExtractSubvectorFromShuffleVector(N, DAG, TLI, LegalOperations))
+ return V;
+
V = peekThroughBitcasts(V);
// If the input is a build vector. Try to make a smaller build vector.
@@ -22424,15 +22581,9 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N, const SDLoc &DL) {
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
- SDValue Ops[] = {LHS, RHS};
unsigned Opcode = N->getOpcode();
SDNodeFlags Flags = N->getFlags();
- // See if we can constant fold the vector operation.
- if (SDValue Fold = DAG.FoldConstantArithmetic(Opcode, SDLoc(LHS),
- LHS.getValueType(), Ops))
- return Fold;
-
// Move unary shuffles with identical masks after a vector binop:
// VBinOp (shuffle A, Undef, Mask), (shuffle B, Undef, Mask))
// --> shuffle (VBinOp A, B), Undef, Mask
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 08598eeded7a..5dfb65ef131a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3367,13 +3367,13 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
case ISD::FSHL:
case ISD::FSHR:
- if (TLI.expandFunnelShift(Node, Tmp1, DAG))
- Results.push_back(Tmp1);
+ if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
+ Results.push_back(Expanded);
break;
case ISD::ROTL:
case ISD::ROTR:
- if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG))
- Results.push_back(Tmp1);
+ if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
+ Results.push_back(Expanded);
break;
case ISD::SADDSAT:
case ISD::UADDSAT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 1fa4d88fcb4a..518e525e13d0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1277,8 +1277,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N, bool IsVP) {
SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
// Lower the rotate to shifts and ORs which can be promoted.
- SDValue Res;
- TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
+ SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
ReplaceValueWith(SDValue(N, 0), Res);
return SDValue();
}
@@ -1286,7 +1285,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
SDValue Hi = GetPromotedInteger(N->getOperand(0));
SDValue Lo = GetPromotedInteger(N->getOperand(1));
- SDValue Amount = GetPromotedInteger(N->getOperand(2));
+ SDValue Amt = GetPromotedInteger(N->getOperand(2));
SDLoc DL(N);
EVT OldVT = N->getOperand(0).getValueType();
@@ -1297,21 +1296,20 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
unsigned NewBits = VT.getScalarSizeInBits();
// Amount has to be interpreted modulo the old bit width.
- Amount =
- DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
+ Amt = DAG.getNode(ISD::UREM, DL, VT, Amt, DAG.getConstant(OldBits, DL, VT));
// If the promoted type is twice the size (or more), then we use the
// traditional funnel 'double' shift codegen. This isn't necessary if the
// shift amount is constant.
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
- if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
+ if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
!TLI.isOperationLegalOrCustom(Opcode, VT)) {
SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
- Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
+ Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
if (!IsFSHR)
Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
return Res;
@@ -1324,9 +1322,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
// Increase Amount to shift the result into the lower bits of the promoted
// type.
if (IsFSHR)
- Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
+ Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, ShiftOffset);
- return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
+ return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
}
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 98312f91d8c0..03dcd0f6d2c9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -83,7 +83,7 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
SDValue Res(&Node, i);
bool Failed = false;
// Don't create a value in map.
- auto ResId = (ValueToIdMap.count(Res)) ? ValueToIdMap[Res] : 0;
+ auto ResId = ValueToIdMap.lookup(Res);
unsigned Mapped = 0;
if (ResId && (ReplacedValues.find(ResId) != ReplacedValues.end())) {
@@ -301,7 +301,7 @@ ScanOperands:
if (IgnoreNodeResults(N->getOperand(i).getNode()))
continue;
- const auto Op = N->getOperand(i);
+ const auto &Op = N->getOperand(i);
LLVM_DEBUG(dbgs() << "Analyzing operand: "; Op.dump(&DAG));
EVT OpVT = Op.getValueType();
switch (getTypeAction(OpVT)) {
@@ -1007,11 +1007,7 @@ SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
///
/// ValVT is the type of values that produced the boolean.
SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
- SDLoc dl(Bool);
- EVT BoolVT = getSetCCResultType(ValVT);
- ISD::NodeType ExtendCode =
- TargetLowering::getExtendForContent(TLI.getBooleanContents(ValVT));
- return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
+ return TLI.promoteTargetBoolean(DAG, Bool, ValVT);
}
/// Return the lower LoVT bits of Op in Lo and the upper HiVT bits in Hi.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 88a28a3be53e..1493f36fcd3e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -254,69 +254,6 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
- if (Op.getOpcode() == ISD::LOAD) {
- LoadSDNode *LD = cast<LoadSDNode>(Node);
- ISD::LoadExtType ExtType = LD->getExtensionType();
- if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
- LLVM_DEBUG(dbgs() << "\nLegalizing extending vector load: ";
- Node->dump(&DAG));
- switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
- LD->getMemoryVT())) {
- default: llvm_unreachable("This action is not supported yet!");
- case TargetLowering::Legal:
- return TranslateLegalizeResults(Op, Node);
- case TargetLowering::Custom: {
- SmallVector<SDValue, 2> ResultVals;
- if (LowerOperationWrapper(Node, ResultVals)) {
- if (ResultVals.empty())
- return TranslateLegalizeResults(Op, Node);
-
- Changed = true;
- return RecursivelyLegalizeResults(Op, ResultVals);
- }
- LLVM_FALLTHROUGH;
- }
- case TargetLowering::Expand: {
- Changed = true;
- std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
- AddLegalizedOperand(Op.getValue(0), Tmp.first);
- AddLegalizedOperand(Op.getValue(1), Tmp.second);
- return Op.getResNo() ? Tmp.first : Tmp.second;
- }
- }
- }
- } else if (Op.getOpcode() == ISD::STORE) {
- StoreSDNode *ST = cast<StoreSDNode>(Node);
- EVT StVT = ST->getMemoryVT();
- MVT ValVT = ST->getValue().getSimpleValueType();
- if (StVT.isVector() && ST->isTruncatingStore()) {
- LLVM_DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
- Node->dump(&DAG));
- switch (TLI.getTruncStoreAction(ValVT, StVT)) {
- default: llvm_unreachable("This action is not supported yet!");
- case TargetLowering::Legal:
- return TranslateLegalizeResults(Op, Node);
- case TargetLowering::Custom: {
- SmallVector<SDValue, 1> ResultVals;
- if (LowerOperationWrapper(Node, ResultVals)) {
- if (ResultVals.empty())
- return TranslateLegalizeResults(Op, Node);
-
- Changed = true;
- return RecursivelyLegalizeResults(Op, ResultVals);
- }
- LLVM_FALLTHROUGH;
- }
- case TargetLowering::Expand: {
- Changed = true;
- SDValue Chain = ExpandStore(Node);
- AddLegalizedOperand(Op, Chain);
- return Chain;
- }
- }
- }
- }
-
bool HasVectorValueOrOp =
llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
llvm::any_of(Node->op_values(),
@@ -329,6 +266,22 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
switch (Op.getOpcode()) {
default:
return TranslateLegalizeResults(Op, Node);
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ ISD::LoadExtType ExtType = LD->getExtensionType();
+ EVT LoadedVT = LD->getMemoryVT();
+ if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
+ Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
+ break;
+ }
+ case ISD::STORE: {
+ StoreSDNode *ST = cast<StoreSDNode>(Node);
+ EVT StVT = ST->getMemoryVT();
+ MVT ValVT = ST->getValue().getSimpleValueType();
+ if (StVT.isVector() && ST->isTruncatingStore())
+ Action = TLI.getTruncStoreAction(ValVT, StVT);
+ break;
+ }
case ISD::MERGE_VALUES:
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
// This operation lies about being legal: when it claims to be legal,
@@ -512,6 +465,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
switch (Action) {
default: llvm_unreachable("This action is not supported yet!");
case TargetLowering::Promote:
+ assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
+ "This action is not supported yet!");
LLVM_DEBUG(dbgs() << "Promoting\n");
Promote(Node, ResultVals);
assert(!ResultVals.empty() && "No results for promotion?");
@@ -731,8 +686,16 @@ SDValue VectorLegalizer::ExpandStore(SDNode *N) {
}
void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
- SDValue Tmp;
switch (Node->getOpcode()) {
+ case ISD::LOAD: {
+ std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
+ Results.push_back(Tmp.first);
+ Results.push_back(Tmp.second);
+ return;
+ }
+ case ISD::STORE:
+ Results.push_back(ExpandStore(Node));
+ return;
case ISD::MERGE_VALUES:
for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
Results.push_back(Node->getOperand(i));
@@ -804,15 +767,15 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
break;
case ISD::FSHL:
case ISD::FSHR:
- if (TLI.expandFunnelShift(Node, Tmp, DAG)) {
- Results.push_back(Tmp);
+ if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
+ Results.push_back(Expanded);
return;
}
break;
case ISD::ROTL:
case ISD::ROTR:
- if (TLI.expandROT(Node, false /*AllowVectorOps*/, Tmp, DAG)) {
- Results.push_back(Tmp);
+ if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
+ Results.push_back(Expanded);
return;
}
break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
index 2695ed36991c..3d5c4c5b1cae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
@@ -168,10 +168,9 @@ void ResourcePriorityQueue::initNodes(std::vector<SUnit> &sunits) {
SUnits = &sunits;
NumNodesSolelyBlocking.resize(SUnits->size(), 0);
- for (unsigned i = 0, e = SUnits->size(); i != e; ++i) {
- SUnit *SU = &(*SUnits)[i];
- initNumRegDefsLeft(SU);
- SU->NodeQueueId = 0;
+ for (SUnit &SU : *SUnits) {
+ initNumRegDefsLeft(&SU);
+ SU.NodeQueueId = 0;
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 84e6d2a16422..aec2cf38b400 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -442,33 +442,32 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
bool UnitLatencies = forceUnitLatencies();
// Pass 2: add the preds, succs, etc.
- for (unsigned su = 0, e = SUnits.size(); su != e; ++su) {
- SUnit *SU = &SUnits[su];
- SDNode *MainNode = SU->getNode();
+ for (SUnit &SU : SUnits) {
+ SDNode *MainNode = SU.getNode();
if (MainNode->isMachineOpcode()) {
unsigned Opc = MainNode->getMachineOpcode();
const MCInstrDesc &MCID = TII->get(Opc);
for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
- SU->isTwoAddress = true;
+ SU.isTwoAddress = true;
break;
}
}
if (MCID.isCommutable())
- SU->isCommutable = true;
+ SU.isCommutable = true;
}
// Find all predecessors and successors of the group.
- for (SDNode *N = SU->getNode(); N; N = N->getGluedNode()) {
+ for (SDNode *N = SU.getNode(); N; N = N->getGluedNode()) {
if (N->isMachineOpcode() &&
TII->get(N->getMachineOpcode()).getImplicitDefs()) {
- SU->hasPhysRegClobbers = true;
+ SU.hasPhysRegClobbers = true;
unsigned NumUsed = InstrEmitter::CountResults(N);
while (NumUsed != 0 && !N->hasAnyUseOfValue(NumUsed - 1))
--NumUsed; // Skip over unused values at the end.
if (NumUsed > TII->get(N->getMachineOpcode()).getNumDefs())
- SU->hasPhysRegDefs = true;
+ SU.hasPhysRegDefs = true;
}
for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
@@ -477,7 +476,8 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
if (isPassiveNode(OpN)) continue; // Not scheduled.
SUnit *OpSU = &SUnits[OpN->getNodeId()];
assert(OpSU && "Node has no SUnit!");
- if (OpSU == SU) continue; // In the same group.
+ if (OpSU == &SU)
+ continue; // In the same group.
EVT OpVT = N->getOperand(i).getValueType();
assert(OpVT != MVT::Glue && "Glued nodes should be in same sunit!");
@@ -508,10 +508,10 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
Dep.setLatency(OpLatency);
if (!isChain && !UnitLatencies) {
computeOperandLatency(OpN, N, i, Dep);
- ST.adjustSchedDependency(OpSU, DefIdx, SU, i, Dep);
+ ST.adjustSchedDependency(OpSU, DefIdx, &SU, i, Dep);
}
- if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
+ if (!SU.addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
// Multiple register uses are combined in the same SUnit. For example,
// we could have a set of glued nodes with all their defs consumed by
// another set of glued nodes. Register pressure tracking sees this as
@@ -721,10 +721,7 @@ void ScheduleDAGSDNodes::dumpSchedule() const {
///
void ScheduleDAGSDNodes::VerifyScheduledSequence(bool isBottomUp) {
unsigned ScheduledNodes = ScheduleDAG::VerifyScheduledDAG(isBottomUp);
- unsigned Noops = 0;
- for (unsigned i = 0, e = Sequence.size(); i != e; ++i)
- if (!Sequence[i])
- ++Noops;
+ unsigned Noops = llvm::count(Sequence, nullptr);
assert(Sequence.size() - Noops == ScheduledNodes &&
"The number of nodes scheduled doesn't match the expected number!");
}
@@ -911,8 +908,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
}
}
- for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
- SUnit *SU = Sequence[i];
+ for (SUnit *SU : Sequence) {
if (!SU) {
// Null SUnit* is a noop.
TII->insertNoop(*Emitter.getBlock(), InsertPos);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
index 540a6e3efbe1..10940478010e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
@@ -169,11 +169,11 @@ void ScheduleDAGVLIW::listScheduleTopDown() {
releaseSuccessors(&EntrySU);
// All leaves to AvailableQueue.
- for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+ for (SUnit &SU : SUnits) {
// It is available if it has no predecessors.
- if (SUnits[i].Preds.empty()) {
- AvailableQueue->push(&SUnits[i]);
- SUnits[i].isAvailable = true;
+ if (SU.Preds.empty()) {
+ AvailableQueue->push(&SU);
+ SU.isAvailable = true;
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c282e03387dd..2ae0d4df7b77 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2499,7 +2499,8 @@ bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
/// sense to specify which elements are demanded or undefined, therefore
/// they are simply ignored.
bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
- APInt &UndefElts, unsigned Depth) {
+ APInt &UndefElts, unsigned Depth) const {
+ unsigned Opcode = V.getOpcode();
EVT VT = V.getValueType();
assert(VT.isVector() && "Vector type expected");
@@ -2511,7 +2512,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
// Deal with some common cases here that work for both fixed and scalable
// vector types.
- switch (V.getOpcode()) {
+ switch (Opcode) {
case ISD::SPLAT_VECTOR:
UndefElts = V.getOperand(0).isUndef()
? APInt::getAllOnes(DemandedElts.getBitWidth())
@@ -2537,7 +2538,12 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
- }
+ default:
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
+ Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
+ return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, Depth);
+ break;
+}
// We don't support other cases than those above for scalable vectors at
// the moment.
@@ -2548,7 +2554,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
UndefElts = APInt::getZero(NumElts);
- switch (V.getOpcode()) {
+ switch (Opcode) {
case ISD::BUILD_VECTOR: {
SDValue Scl;
for (unsigned i = 0; i != NumElts; ++i) {
@@ -2600,13 +2606,30 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
}
break;
}
+ case ISD::ANY_EXTEND_VECTOR_INREG:
+ case ISD::SIGN_EXTEND_VECTOR_INREG:
+ case ISD::ZERO_EXTEND_VECTOR_INREG: {
+ // Widen the demanded elts by the src element count.
+ SDValue Src = V.getOperand(0);
+ // We don't support scalable vectors at the moment.
+ if (Src.getValueType().isScalableVector())
+ return false;
+ unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+ APInt UndefSrcElts;
+ APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
+ if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
+ UndefElts = UndefSrcElts.truncOrSelf(NumElts);
+ return true;
+ }
+ break;
+ }
}
return false;
}
/// Helper wrapper to main isSplatValue function.
-bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
+bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
EVT VT = V.getValueType();
assert(VT.isVector() && "Vector type expected");
@@ -5291,9 +5314,10 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
if (isUndef(Opcode, Ops))
return getUNDEF(VT);
- // Handle the case of two scalars.
+ // Handle binops special cases.
if (NumOps == 2) {
- // TODO: Move foldConstantFPMath here?
+ if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops[0], Ops[1]))
+ return CFP;
if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
@@ -5463,10 +5487,11 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
// should. That will require dealing with a potentially non-default
// rounding mode, checking the "opStatus" return value from the APFloat
// math calculations, and possibly other variations.
- auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
- auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
+ ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
+ ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
if (N1CFP && N2CFP) {
- APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
+ APFloat C1 = N1CFP->getValueAPF(); // make copy
+ const APFloat &C2 = N2CFP->getValueAPF();
switch (Opcode) {
case ISD::FADD:
C1.add(C2, APFloat::rmNearestTiesToEven);
@@ -5486,6 +5511,14 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
case ISD::FCOPYSIGN:
C1.copySign(C2);
return getConstantFP(C1, DL, VT);
+ case ISD::FMINNUM:
+ return getConstantFP(minnum(C1, C2), DL, VT);
+ case ISD::FMAXNUM:
+ return getConstantFP(maxnum(C1, C2), DL, VT);
+ case ISD::FMINIMUM:
+ return getConstantFP(minimum(C1, C2), DL, VT);
+ case ISD::FMAXIMUM:
+ return getConstantFP(maximum(C1, C2), DL, VT);
default: break;
}
}
@@ -5502,8 +5535,9 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
switch (Opcode) {
case ISD::FSUB:
// -0.0 - undef --> undef (consistent with "fneg undef")
- if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef())
- return getUNDEF(VT);
+ if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
+ if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
+ return getUNDEF(VT);
LLVM_FALLTHROUGH;
case ISD::FADD:
@@ -5962,9 +5996,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
return SV;
- if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
- return V;
-
// Canonicalize an UNDEF to the RHS, even over a constant.
if (N1.isUndef()) {
if (TLI->isCommutativeBinOp(Opcode)) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7726a0007e44..63cd723cf6da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1036,7 +1036,6 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
AA = aa;
GFI = gfi;
LibInfo = li;
- DL = &DAG.getDataLayout();
Context = DAG.getContext();
LPadToCallSiteMap.clear();
SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
@@ -1626,6 +1625,9 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
return getValue(Equiv->getGlobalValue());
+ if (const auto *NC = dyn_cast<NoCFIValue>(C))
+ return getValue(NC->getGlobalValue());
+
VectorType *VecTy = cast<VectorType>(V->getType());
// Now that we know the number and type of the elements, get that number of
@@ -1921,8 +1923,8 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
DAG.getDataLayout().getAllocaAddrSpace()),
PtrValueVTs);
- SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
- DemoteReg, PtrValueVTs[0]);
+ SDValue RetPtr =
+ DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
SDValue RetOp = getValue(I.getOperand(0));
SmallVector<EVT, 4> ValueVTs, MemVTs;
@@ -2657,7 +2659,8 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
SDLoc dl = getCurSDLoc();
SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
const Module &M = *ParentBB->getParent()->getFunction().getParent();
- Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
+ Align Align =
+ DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
// Generate code to load the content of the guard slot.
SDValue GuardVal = DAG.getLoad(
@@ -3058,14 +3061,14 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
MachineBasicBlock *Last) {
// Update JTCases.
- for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
- if (SL->JTCases[i].first.HeaderBB == First)
- SL->JTCases[i].first.HeaderBB = Last;
+ for (JumpTableBlock &JTB : SL->JTCases)
+ if (JTB.first.HeaderBB == First)
+ JTB.first.HeaderBB = Last;
// Update BitTestCases.
- for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
- if (SL->BitTestCases[i].Parent == First)
- SL->BitTestCases[i].Parent = Last;
+ for (BitTestBlock &BTB : SL->BitTestCases)
+ if (BTB.Parent == First)
+ BTB.Parent = Last;
}
void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
@@ -3111,6 +3114,8 @@ void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
SDNodeFlags Flags;
+ if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
+ Flags.copyFMF(*FPOp);
SDValue Op = getValue(I.getOperand(0));
SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
@@ -3881,7 +3886,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
if (Field) {
// N = N + Offset
- uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
+ uint64_t Offset =
+ DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
// In an inbounds GEP with an offset that is nonnegative even when
// interpreted as signed, assume there is no unsigned overflow.
@@ -3898,7 +3904,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
// (and fix up the result later).
unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
MVT IdxTy = MVT::getIntegerVT(IdxSize);
- TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
+ TypeSize ElementSize =
+ DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
// We intentionally mask away the high bits here; ElementSize may not
// fit in IdxTy.
APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
@@ -4788,7 +4795,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
}
// Use TargetConstant instead of a regular constant for immarg.
- EVT VT = TLI.getValueType(*DL, Arg->getType(), true);
+ EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
assert(CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled");
@@ -6571,7 +6578,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
} else {
EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
const Value *Global = TLI.getSDagStackGuard(M);
- Align Align = DL->getPrefTypeAlign(Global->getType());
+ Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
MachinePointerInfo(Global, 0), Align,
MachineMemOperand::MOVolatile);
@@ -7127,12 +7134,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
}
SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
SDValue VectorInduction = DAG.getNode(
- ISD::UADDO, sdl, DAG.getVTList(VecTy, CCVT), VectorIndex, VectorStep);
- SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction.getValue(0),
+ ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
+ SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
VectorTripCount, ISD::CondCode::SETULT);
- setValue(&I, DAG.getNode(ISD::AND, sdl, CCVT,
- DAG.getNOT(sdl, VectorInduction.getValue(1), CCVT),
- SetCC));
+ setValue(&I, SetCC);
return;
}
case Intrinsic::experimental_vector_insert: {
@@ -7317,32 +7322,26 @@ static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
SmallVector<SDValue, 7> &OpValues,
- bool isGather) {
+ bool IsGather) {
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Value *PtrOperand = VPIntrin.getArgOperand(0);
- MaybeAlign Alignment = DAG.getEVTAlign(VT);
+ MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT);
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
SDValue LD;
bool AddToChain = true;
- if (!isGather) {
+ if (!IsGather) {
// Do not serialize variable-length loads of constant memory with
// anything.
- MemoryLocation ML;
- if (VT.isScalableVector())
- ML = MemoryLocation::getAfter(PtrOperand);
- else
- ML = MemoryLocation(
- PtrOperand,
- LocationSize::precise(
- DAG.getDataLayout().getTypeStoreSize(VPIntrin.getType())),
- AAInfo);
+ MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
AddToChain = !AA || !AA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);
+ MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
} else {
@@ -7380,18 +7379,20 @@ void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
SmallVector<SDValue, 7> &OpValues,
- bool isScatter) {
+ bool IsScatter) {
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Value *PtrOperand = VPIntrin.getArgOperand(1);
EVT VT = OpValues[0].getValueType();
- MaybeAlign Alignment = DAG.getEVTAlign(VT);
+ MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+ if (!Alignment)
+ Alignment = DAG.getEVTAlign(VT);
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
SDValue ST;
- if (!isScatter) {
+ if (!IsScatter) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
- VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);
+ MemoryLocation::UnknownSize, *Alignment, AAInfo);
ST =
DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO, false /* IsTruncating */);
@@ -7690,8 +7691,9 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
PointerType::getUnqual(LoadTy));
- if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
- const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
+ if (const Constant *LoadCst =
+ ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
+ LoadTy, Builder.DAG.getDataLayout()))
return Builder.getValue(LoadCst);
}
@@ -9646,8 +9648,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
// We push in swifterror return as the last element of CLI.Ins.
ArgListTy &Args = CLI.getArgs();
if (supportSwiftError()) {
- for (unsigned i = 0, e = Args.size(); i != e; ++i) {
- if (Args[i].IsSwiftError) {
+ for (const ArgListEntry &Arg : Args) {
+ if (Arg.IsSwiftError) {
ISD::InputArg MyFlags;
MyFlags.VT = getPointerTy(DL);
MyFlags.ArgVT = EVT(getPointerTy(DL));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index d6122aa0a739..ea48042a5dcf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -190,7 +190,6 @@ public:
static const unsigned LowestSDNodeOrder = 1;
SelectionDAG &DAG;
- const DataLayout *DL = nullptr;
AAResults *AA = nullptr;
const TargetLibraryInfo *LibInfo;
@@ -568,9 +567,9 @@ private:
void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
void visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
- SmallVector<SDValue, 7> &OpValues, bool isGather);
+ SmallVector<SDValue, 7> &OpValues, bool IsGather);
void visitVPStoreScatter(const VPIntrinsic &VPIntrin,
- SmallVector<SDValue, 7> &OpValues, bool isScatter);
+ SmallVector<SDValue, 7> &OpValues, bool IsScatter);
void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
void visitVAStart(const CallInst &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index c7e37cf8ca14..77e11b364588 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -297,7 +297,7 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
#ifndef NDEBUG
dbgs() << "If a target marks an instruction with "
"'usesCustomInserter', it must implement "
- "TargetLowering::EmitInstrWithCustomInserter!";
+ "TargetLowering::EmitInstrWithCustomInserter!\n";
#endif
llvm_unreachable(nullptr);
}
@@ -1784,27 +1784,25 @@ SelectionDAGISel::FinishBasicBlock() {
}
// Update PHI Nodes
- for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
- pi != pe; ++pi) {
- MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
+ for (const std::pair<MachineInstr *, unsigned> &P :
+ FuncInfo->PHINodesToUpdate) {
+ MachineInstrBuilder PHI(*MF, P.first);
MachineBasicBlock *PHIBB = PHI->getParent();
assert(PHI->isPHI() &&
"This is not a machine PHI node that we are updating!");
// This is "default" BB. We have two jumps to it. From "header" BB and
// from last "case" BB, unless the latter was skipped.
if (PHIBB == BTB.Default) {
- PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
+ PHI.addReg(P.second).addMBB(BTB.Parent);
if (!BTB.ContiguousRange) {
- PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
- .addMBB(BTB.Cases.back().ThisBB);
+ PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
}
}
// One of "cases" BB.
- for (unsigned j = 0, ej = BTB.Cases.size();
- j != ej; ++j) {
- MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
+ for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
+ MachineBasicBlock* cBB = BT.ThisBB;
if (cBB->isSuccessor(PHIBB))
- PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
+ PHI.addReg(P.second).addMBB(cBB);
}
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 737695b5eabe..e6b06ab93d6b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3136,6 +3136,19 @@ bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
return false;
}
+bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
+ APInt &UndefElts,
+ unsigned Depth) const {
+ assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
+ Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
+ Op.getOpcode() == ISD::INTRINSIC_VOID) &&
+ "Should use isSplatValue if you don't know whether Op"
+ " is a target node!");
+ return false;
+}
+
// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
// work with truncating build vectors and vectors with elements of less than
// 8 bits.
@@ -4853,13 +4866,9 @@ TargetLowering::ParseConstraints(const DataLayout &DL,
}
// Now select chosen alternative in each constraint.
- for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
- cIndex != eIndex; ++cIndex) {
- AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
- if (cInfo.Type == InlineAsm::isClobber)
- continue;
- cInfo.selectAlternative(bestMAIndex);
- }
+ for (AsmOperandInfo &cInfo : ConstraintOperands)
+ if (cInfo.Type != InlineAsm::isClobber)
+ cInfo.selectAlternative(bestMAIndex);
}
}
@@ -4927,9 +4936,9 @@ TargetLowering::ConstraintWeight
ConstraintWeight BestWeight = CW_Invalid;
// Loop over the options, keeping track of the most general one.
- for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
+ for (const std::string &rCode : *rCodes) {
ConstraintWeight weight =
- getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
+ getSingleConstraintMatchWeight(info, rCode.c_str());
if (weight > BestWeight)
BestWeight = weight;
}
@@ -6550,15 +6559,15 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
true);
}
-bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
- SelectionDAG &DAG) const {
+SDValue TargetLowering::expandFunnelShift(SDNode *Node,
+ SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
!isOperationLegalOrCustom(ISD::SRL, VT) ||
!isOperationLegalOrCustom(ISD::SUB, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
- return false;
+ return SDValue();
SDValue X = Node->getOperand(0);
SDValue Y = Node->getOperand(1);
@@ -6592,8 +6601,7 @@ bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
}
Z = DAG.getNOT(DL, Z, ShVT);
}
- Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
- return true;
+ return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
}
SDValue ShX, ShY;
@@ -6633,13 +6641,12 @@ bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
}
}
- Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
- return true;
+ return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
}
// TODO: Merge with expandFunnelShift.
-bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
- SDValue &Result, SelectionDAG &DAG) const {
+SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
+ SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
unsigned EltSizeInBits = VT.getScalarSizeInBits();
bool IsLeft = Node->getOpcode() == ISD::ROTL;
@@ -6650,12 +6657,12 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
EVT ShVT = Op1.getValueType();
SDValue Zero = DAG.getConstant(0, DL, ShVT);
- // If a rotate in the other direction is supported, use it.
+ // If a rotate in the other direction is more supported, use it.
unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
- if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
+ if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
+ isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
- Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
- return true;
+ return DAG.getNode(RevRot, DL, VT, Op0, Sub);
}
if (!AllowVectorOps && VT.isVector() &&
@@ -6664,7 +6671,7 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
!isOperationLegalOrCustom(ISD::SUB, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
!isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
- return false;
+ return SDValue();
unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
@@ -6690,8 +6697,7 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
HsVal =
DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
}
- Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
- return true;
+ return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
}
void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
@@ -8048,7 +8054,8 @@ SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
return DAG.UnrollVectorOp(Node);
- SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
+ EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
return DAG.getSelect(DL, VT, Cond, Op0, Op1);
}
generated by cgit v1.3 (git 2.53.0) at 2026-07-29 16:16:36 +0000