summaryrefslogtreecommitdiff
path: root/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp')
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp265
1 files changed, 161 insertions, 104 deletions
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7643790df350..67928d4bdbd5 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -32,7 +32,6 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/MachineValueType.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetLowering.h"
@@ -41,6 +40,7 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MachineValueType.h"
#include "llvm/Support/MathExtras.h"
#include <cassert>
#include <cstdint>
@@ -63,7 +63,7 @@ class VectorLegalizer {
/// legalizing the same thing more than once.
SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
- /// \brief Adds a node to the translation cache.
+ /// Adds a node to the translation cache.
void AddLegalizedOperand(SDValue From, SDValue To) {
LegalizedNodes.insert(std::make_pair(From, To));
// If someone requests legalization of the new node, return itself.
@@ -71,55 +71,55 @@ class VectorLegalizer {
LegalizedNodes.insert(std::make_pair(To, To));
}
- /// \brief Legalizes the given node.
+ /// Legalizes the given node.
SDValue LegalizeOp(SDValue Op);
- /// \brief Assuming the node is legal, "legalize" the results.
+ /// Assuming the node is legal, "legalize" the results.
SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
- /// \brief Implements unrolling a VSETCC.
+ /// Implements unrolling a VSETCC.
SDValue UnrollVSETCC(SDValue Op);
- /// \brief Implement expand-based legalization of vector operations.
+ /// Implement expand-based legalization of vector operations.
///
/// This is just a high-level routine to dispatch to specific code paths for
/// operations to legalize them.
SDValue Expand(SDValue Op);
- /// \brief Implements expansion for FNEG; falls back to UnrollVectorOp if
+ /// Implements expansion for FNEG; falls back to UnrollVectorOp if
/// FSUB isn't legal.
///
/// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
/// SINT_TO_FLOAT and SHR on vectors isn't legal.
SDValue ExpandUINT_TO_FLOAT(SDValue Op);
- /// \brief Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
+ /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
SDValue ExpandSEXTINREG(SDValue Op);
- /// \brief Implement expansion for ANY_EXTEND_VECTOR_INREG.
+ /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
///
/// Shuffles the low lanes of the operand into place and bitcasts to the proper
/// type. The contents of the bits in the extended part of each element are
/// undef.
SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op);
- /// \brief Implement expansion for SIGN_EXTEND_VECTOR_INREG.
+ /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
///
/// Shuffles the low lanes of the operand into place, bitcasts to the proper
/// type, then shifts left and arithmetic shifts right to introduce a sign
/// extension.
SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op);
- /// \brief Implement expansion for ZERO_EXTEND_VECTOR_INREG.
+ /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
///
/// Shuffles the low lanes of the operand into place and blends zeros into
/// the remaining lanes, finally bitcasting to the proper type.
SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
- /// \brief Expand bswap of vectors into a shuffle if legal.
+ /// Expand bswap of vectors into a shuffle if legal.
SDValue ExpandBSWAP(SDValue Op);
- /// \brief Implement vselect in terms of XOR, AND, OR when blend is not
+ /// Implement vselect in terms of XOR, AND, OR when blend is not
/// supported by the target.
SDValue ExpandVSELECT(SDValue Op);
SDValue ExpandSELECT(SDValue Op);
@@ -130,29 +130,30 @@ class VectorLegalizer {
SDValue ExpandBITREVERSE(SDValue Op);
SDValue ExpandCTLZ(SDValue Op);
SDValue ExpandCTTZ_ZERO_UNDEF(SDValue Op);
-
- /// \brief Implements vector promotion.
+ SDValue ExpandStrictFPOp(SDValue Op);
+
+ /// Implements vector promotion.
///
/// This is essentially just bitcasting the operands to a different type and
/// bitcasting the result back to the original type.
SDValue Promote(SDValue Op);
- /// \brief Implements [SU]INT_TO_FP vector promotion.
+ /// Implements [SU]INT_TO_FP vector promotion.
///
- /// This is a [zs]ext of the input operand to the next size up.
+ /// This is a [zs]ext of the input operand to a larger integer type.
SDValue PromoteINT_TO_FP(SDValue Op);
- /// \brief Implements FP_TO_[SU]INT vector promotion of the result type.
+ /// Implements FP_TO_[SU]INT vector promotion of the result type.
///
- /// It is promoted to the next size up integer type. The result is then
+ /// It is promoted to a larger integer type. The result is then
/// truncated back to the original type.
- SDValue PromoteFP_TO_INT(SDValue Op, bool isSigned);
+ SDValue PromoteFP_TO_INT(SDValue Op);
public:
VectorLegalizer(SelectionDAG& dag) :
DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
- /// \brief Begin legalizer the vector operations in the DAG.
+ /// Begin legalizer the vector operations in the DAG.
bool Run();
};
@@ -222,14 +223,16 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
for (const SDValue &Op : Node->op_values())
Ops.push_back(LegalizeOp(Op));
- SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0);
+ SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops),
+ Op.getResNo());
bool HasVectorValue = false;
if (Op.getOpcode() == ISD::LOAD) {
LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
ISD::LoadExtType ExtType = LD->getExtensionType();
if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
- DEBUG(dbgs() << "\nLegalizing extending vector load: "; Node->dump(&DAG));
+ LLVM_DEBUG(dbgs() << "\nLegalizing extending vector load: ";
+ Node->dump(&DAG));
switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
LD->getMemoryVT())) {
default: llvm_unreachable("This action is not supported yet!");
@@ -261,8 +264,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
EVT StVT = ST->getMemoryVT();
MVT ValVT = ST->getValue().getSimpleValueType();
if (StVT.isVector() && ST->isTruncatingStore()) {
- DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
- Node->dump(&DAG));
+ LLVM_DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
+ Node->dump(&DAG));
switch (TLI.getTruncStoreAction(ValVT, StVT)) {
default: llvm_unreachable("This action is not supported yet!");
case TargetLowering::Legal:
@@ -287,10 +290,34 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
if (!HasVectorValue)
return TranslateLegalizeResults(Op, Result);
- EVT QueryType;
+ TargetLowering::LegalizeAction Action = TargetLowering::Legal;
switch (Op.getOpcode()) {
default:
return TranslateLegalizeResults(Op, Result);
+ case ISD::STRICT_FADD:
+ case ISD::STRICT_FSUB:
+ case ISD::STRICT_FMUL:
+ case ISD::STRICT_FDIV:
+ case ISD::STRICT_FSQRT:
+ case ISD::STRICT_FMA:
+ case ISD::STRICT_FPOW:
+ case ISD::STRICT_FPOWI:
+ case ISD::STRICT_FSIN:
+ case ISD::STRICT_FCOS:
+ case ISD::STRICT_FEXP:
+ case ISD::STRICT_FEXP2:
+ case ISD::STRICT_FLOG:
+ case ISD::STRICT_FLOG10:
+ case ISD::STRICT_FLOG2:
+ case ISD::STRICT_FRINT:
+ case ISD::STRICT_FNEARBYINT:
+ // These pseudo-ops get legalized as if they were their non-strict
+ // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
+ // is also legal, but if ISD::FSQRT requires expansion then so does
+ // ISD::STRICT_FSQRT.
+ Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
+ Node->getValueType(0));
+ break;
case ISD::ADD:
case ISD::SUB:
case ISD::MUL:
@@ -366,42 +393,47 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::UMAX:
case ISD::SMUL_LOHI:
case ISD::UMUL_LOHI:
- QueryType = Node->getValueType(0);
+ case ISD::FCANONICALIZE:
+ Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
case ISD::FP_ROUND_INREG:
- QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT();
+ Action = TLI.getOperationAction(Node->getOpcode(),
+ cast<VTSDNode>(Node->getOperand(1))->getVT());
break;
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP:
- QueryType = Node->getOperand(0).getValueType();
+ Action = TLI.getOperationAction(Node->getOpcode(),
+ Node->getOperand(0).getValueType());
break;
case ISD::MSCATTER:
- QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
+ Action = TLI.getOperationAction(Node->getOpcode(),
+ cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
break;
case ISD::MSTORE:
- QueryType = cast<MaskedStoreSDNode>(Node)->getValue().getValueType();
+ Action = TLI.getOperationAction(Node->getOpcode(),
+ cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
break;
}
- DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
+ LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
- switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
+ switch (Action) {
default: llvm_unreachable("This action is not supported yet!");
case TargetLowering::Promote:
Result = Promote(Op);
Changed = true;
break;
case TargetLowering::Legal:
- DEBUG(dbgs() << "Legal node: nothing to do\n");
+ LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
break;
case TargetLowering::Custom: {
- DEBUG(dbgs() << "Trying custom legalization\n");
+ LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
if (SDValue Tmp1 = TLI.LowerOperation(Op, DAG)) {
- DEBUG(dbgs() << "Successfully custom legalized node\n");
+ LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
Result = Tmp1;
break;
}
- DEBUG(dbgs() << "Could not custom legalize node\n");
+ LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
LLVM_FALLTHROUGH;
}
case TargetLowering::Expand:
@@ -431,7 +463,7 @@ SDValue VectorLegalizer::Promote(SDValue Op) {
case ISD::FP_TO_UINT:
case ISD::FP_TO_SINT:
// Promote the operation by extending the operand.
- return PromoteFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT);
+ return PromoteFP_TO_INT(Op);
}
// There are currently two cases of vector promotion:
@@ -472,20 +504,11 @@ SDValue VectorLegalizer::Promote(SDValue Op) {
SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
// INT_TO_FP operations may require the input operand be promoted even
// when the type is otherwise legal.
- EVT VT = Op.getOperand(0).getValueType();
- assert(Op.getNode()->getNumValues() == 1 &&
- "Can't promote a vector with multiple results!");
-
- // Normal getTypeToPromoteTo() doesn't work here, as that will promote
- // by widening the vector w/ the same element width and twice the number
- // of elements. We want the other way around, the same number of elements,
- // each twice the width.
- //
- // Increase the bitwidth of the element to the next pow-of-two
- // (which is greater than 8 bits).
+ MVT VT = Op.getOperand(0).getSimpleValueType();
+ MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT);
+ assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
+ "Vectors have different number of elements!");
- EVT NVT = VT.widenIntegerVectorElementType(*DAG.getContext());
- assert(NVT.isSimple() && "Promoting to a non-simple vector type!");
SDLoc dl(Op);
SmallVector<SDValue, 4> Operands(Op.getNumOperands());
@@ -505,35 +528,28 @@ SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
// elements and then truncate the result. This is different from the default
// PromoteVector which uses bitcast to promote thus assumning that the
// promoted vector type has the same overall size.
-SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op, bool isSigned) {
- assert(Op.getNode()->getNumValues() == 1 &&
- "Can't promote a vector with multiple results!");
- EVT VT = Op.getValueType();
+SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op) {
+ MVT VT = Op.getSimpleValueType();
+ MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT);
+ assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
+ "Vectors have different number of elements!");
- EVT NewVT = VT;
- unsigned NewOpc;
- while (true) {
- NewVT = NewVT.widenIntegerVectorElementType(*DAG.getContext());
- assert(NewVT.isSimple() && "Promoting to a non-simple vector type!");
- if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) {
- NewOpc = ISD::FP_TO_SINT;
- break;
- }
- if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) {
- NewOpc = ISD::FP_TO_UINT;
- break;
- }
- }
+ unsigned NewOpc = Op->getOpcode();
+ // Change FP_TO_UINT to FP_TO_SINT if possible.
+ // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
+ if (NewOpc == ISD::FP_TO_UINT &&
+ TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
+ NewOpc = ISD::FP_TO_SINT;
SDLoc dl(Op);
- SDValue Promoted = DAG.getNode(NewOpc, dl, NewVT, Op.getOperand(0));
+ SDValue Promoted = DAG.getNode(NewOpc, dl, NVT, Op.getOperand(0));
// Assert that the converted value fits in the original type. If it doesn't
// (eg: because the value being converted is too big), then the result of the
// original operation was undefined anyway, so the assert is still correct.
Promoted = DAG.getNode(Op->getOpcode() == ISD::FP_TO_UINT ? ISD::AssertZext
: ISD::AssertSext,
- dl, NewVT, Promoted,
+ dl, NVT, Promoted,
DAG.getValueType(VT.getScalarType()));
return DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
}
@@ -665,9 +681,14 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
Value = DAG.getBuildVector(Op.getNode()->getValueType(0), dl, Vals);
} else {
SDValue Scalarized = TLI.scalarizeVectorLoad(LD, DAG);
-
- NewChain = Scalarized.getValue(1);
- Value = Scalarized.getValue(0);
+ // Skip past MERGE_VALUE node if known.
+ if (Scalarized->getOpcode() == ISD::MERGE_VALUES) {
+ NewChain = Scalarized.getOperand(1);
+ Value = Scalarized.getOperand(0);
+ } else {
+ NewChain = Scalarized.getValue(1);
+ Value = Scalarized.getValue(0);
+ }
}
AddLegalizedOperand(Op.getValue(0), Value);
@@ -678,35 +699,6 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
SDValue VectorLegalizer::ExpandStore(SDValue Op) {
StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
-
- EVT StVT = ST->getMemoryVT();
- EVT MemSclVT = StVT.getScalarType();
- unsigned ScalarSize = MemSclVT.getSizeInBits();
-
- // Round odd types to the next pow of two.
- if (!isPowerOf2_32(ScalarSize)) {
- // FIXME: This is completely broken and inconsistent with ExpandLoad
- // handling.
-
- // For sub-byte element sizes, this ends up with 0 stride between elements,
- // so the same element just gets re-written to the same location. There seem
- // to be tests explicitly testing for this broken behavior though. tests
- // for this broken behavior.
-
- LLVMContext &Ctx = *DAG.getContext();
-
- EVT NewMemVT
- = EVT::getVectorVT(Ctx,
- MemSclVT.getIntegerVT(Ctx, NextPowerOf2(ScalarSize)),
- StVT.getVectorNumElements());
-
- SDValue NewVectorStore = DAG.getTruncStore(
- ST->getChain(), SDLoc(Op), ST->getValue(), ST->getBasePtr(),
- ST->getPointerInfo(), NewMemVT, ST->getAlignment(),
- ST->getMemOperand()->getFlags(), ST->getAAInfo());
- ST = cast<StoreSDNode>(NewVectorStore.getNode());
- }
-
SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
AddLegalizedOperand(Op, TF);
return TF;
@@ -743,6 +735,24 @@ SDValue VectorLegalizer::Expand(SDValue Op) {
return ExpandCTLZ(Op);
case ISD::CTTZ_ZERO_UNDEF:
return ExpandCTTZ_ZERO_UNDEF(Op);
+ case ISD::STRICT_FADD:
+ case ISD::STRICT_FSUB:
+ case ISD::STRICT_FMUL:
+ case ISD::STRICT_FDIV:
+ case ISD::STRICT_FSQRT:
+ case ISD::STRICT_FMA:
+ case ISD::STRICT_FPOW:
+ case ISD::STRICT_FPOWI:
+ case ISD::STRICT_FSIN:
+ case ISD::STRICT_FCOS:
+ case ISD::STRICT_FEXP:
+ case ISD::STRICT_FEXP2:
+ case ISD::STRICT_FLOG:
+ case ISD::STRICT_FLOG10:
+ case ISD::STRICT_FLOG2:
+ case ISD::STRICT_FRINT:
+ case ISD::STRICT_FNEARBYINT:
+ return ExpandStrictFPOp(Op);
default:
return DAG.UnrollVectorOp(Op.getNode());
}
@@ -1036,7 +1046,7 @@ SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
// Two to the power of half-word-size.
- SDValue TWOHW = DAG.getConstantFP(1 << (BW / 2), DL, Op.getValueType());
+ SDValue TWOHW = DAG.getConstantFP(1ULL << (BW / 2), DL, Op.getValueType());
// Clear upper part of LO, lower HI
SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord);
@@ -1129,6 +1139,53 @@ SDValue VectorLegalizer::ExpandCTTZ_ZERO_UNDEF(SDValue Op) {
return DAG.UnrollVectorOp(Op.getNode());
}
+SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) {
+ EVT VT = Op.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ unsigned NumElems = VT.getVectorNumElements();
+ unsigned NumOpers = Op.getNumOperands();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT ValueVTs[] = {EltVT, MVT::Other};
+ SDValue Chain = Op.getOperand(0);
+ SDLoc dl(Op);
+
+ SmallVector<SDValue, 32> OpValues;
+ SmallVector<SDValue, 32> OpChains;
+ for (unsigned i = 0; i < NumElems; ++i) {
+ SmallVector<SDValue, 4> Opers;
+ SDValue Idx = DAG.getConstant(i, dl,
+ TLI.getVectorIdxTy(DAG.getDataLayout()));
+
+ // The Chain is the first operand.
+ Opers.push_back(Chain);
+
+ // Now process the remaining operands.
+ for (unsigned j = 1; j < NumOpers; ++j) {
+ SDValue Oper = Op.getOperand(j);
+ EVT OperVT = Oper.getValueType();
+
+ if (OperVT.isVector())
+ Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+ EltVT, Oper, Idx);
+
+ Opers.push_back(Oper);
+ }
+
+ SDValue ScalarOp = DAG.getNode(Op->getOpcode(), dl, ValueVTs, Opers);
+
+ OpValues.push_back(ScalarOp.getValue(0));
+ OpChains.push_back(ScalarOp.getValue(1));
+ }
+
+ SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
+ SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
+
+ AddLegalizedOperand(Op.getValue(0), Result);
+ AddLegalizedOperand(Op.getValue(1), NewChain);
+
+ return NewChain;
+}
+
SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
EVT VT = Op.getValueType();
unsigned NumElems = VT.getVectorNumElements();
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-07 16:02:00 +0000