summaryrefslogtreecommitdiff
path: root/include/llvm/CodeGen/SelectionDAGNodes.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/llvm/CodeGen/SelectionDAGNodes.h')
-rw-r--r--include/llvm/CodeGen/SelectionDAGNodes.h163
1 files changed, 125 insertions, 38 deletions
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 522c2f1b2cb2..1af22185d366 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -31,17 +31,18 @@
#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/MachineValueType.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Operator.h"
#include "llvm/Support/AlignOf.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MachineValueType.h"
#include <algorithm>
#include <cassert>
#include <climits>
@@ -189,8 +190,10 @@ public:
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
- inline void dump(const SelectionDAG *G = nullptr) const;
- inline void dumpr(const SelectionDAG *G = nullptr) const;
+ inline void dump() const;
+ inline void dump(const SelectionDAG *G) const;
+ inline void dumpr() const;
+ inline void dumpr(const SelectionDAG *G) const;
/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
@@ -357,21 +360,34 @@ private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
- bool UnsafeAlgebra : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool VectorReduction : 1;
bool AllowContract : 1;
+ bool ApproximateFuncs : 1;
+ bool AllowReassociation : 1;
public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
: AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
- Exact(false), UnsafeAlgebra(false), NoNaNs(false), NoInfs(false),
+ Exact(false), NoNaNs(false), NoInfs(false),
NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
- AllowContract(false) {}
+ AllowContract(false), ApproximateFuncs(false),
+ AllowReassociation(false) {}
+
+ /// Propagate the fast-math-flags from an IR FPMathOperator.
+ void copyFMF(const FPMathOperator &FPMO) {
+ setNoNaNs(FPMO.hasNoNaNs());
+ setNoInfs(FPMO.hasNoInfs());
+ setNoSignedZeros(FPMO.hasNoSignedZeros());
+ setAllowReciprocal(FPMO.hasAllowReciprocal());
+ setAllowContract(FPMO.hasAllowContract());
+ setApproximateFuncs(FPMO.hasApproxFunc());
+ setAllowReassociation(FPMO.hasAllowReassoc());
+ }
/// Sets the state of the flags to the defined state.
void setDefined() { AnyDefined = true; }
@@ -391,10 +407,6 @@ public:
setDefined();
Exact = b;
}
- void setUnsafeAlgebra(bool b) {
- setDefined();
- UnsafeAlgebra = b;
- }
void setNoNaNs(bool b) {
setDefined();
NoNaNs = b;
@@ -419,18 +431,32 @@ public:
setDefined();
AllowContract = b;
}
+ void setApproximateFuncs(bool b) {
+ setDefined();
+ ApproximateFuncs = b;
+ }
+ void setAllowReassociation(bool b) {
+ setDefined();
+ AllowReassociation = b;
+ }
// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
- bool hasUnsafeAlgebra() const { return UnsafeAlgebra; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasVectorReduction() const { return VectorReduction; }
bool hasAllowContract() const { return AllowContract; }
+ bool hasApproximateFuncs() const { return ApproximateFuncs; }
+ bool hasAllowReassociation() const { return AllowReassociation; }
+
+ bool isFast() const {
+ return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs &&
+ AllowContract && ApproximateFuncs && AllowReassociation;
+ }
/// Clear any flags in this flag set that aren't also set in Flags.
/// If the given Flags are undefined then don't do anything.
@@ -440,13 +466,14 @@ public:
NoUnsignedWrap &= Flags.NoUnsignedWrap;
NoSignedWrap &= Flags.NoSignedWrap;
Exact &= Flags.Exact;
- UnsafeAlgebra &= Flags.UnsafeAlgebra;
NoNaNs &= Flags.NoNaNs;
NoInfs &= Flags.NoInfs;
NoSignedZeros &= Flags.NoSignedZeros;
AllowReciprocal &= Flags.AllowReciprocal;
VectorReduction &= Flags.VectorReduction;
AllowContract &= Flags.AllowContract;
+ ApproximateFuncs &= Flags.ApproximateFuncs;
+ AllowReassociation &= Flags.AllowReassociation;
}
};
@@ -466,11 +493,13 @@ protected:
friend class SDNode;
friend class MemIntrinsicSDNode;
friend class MemSDNode;
+ friend class SelectionDAG;
uint16_t HasDebugValue : 1;
uint16_t IsMemIntrinsic : 1;
+ uint16_t IsDivergent : 1;
};
- enum { NumSDNodeBits = 2 };
+ enum { NumSDNodeBits = 3 };
class ConstantSDNodeBitfields {
friend class ConstantSDNode;
@@ -540,7 +569,7 @@ protected:
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
- static_assert(sizeof(LoadSDNodeBitfields) <= 4, "field too wide");
+ static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
private:
@@ -662,6 +691,8 @@ public:
bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
+ bool isDivergent() const { return SDNodeBits.IsDivergent; }
+
/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }
@@ -796,16 +827,44 @@ public:
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
- /// if DAG changes.
+ /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
+ /// giving up. The TopologicalPrune flag signals that positive NodeIds are
+ /// topologically ordered (Operands have strictly smaller node id) and search
+ /// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
SmallPtrSetImpl<const SDNode *> &Visited,
SmallVectorImpl<const SDNode *> &Worklist,
- unsigned int MaxSteps = 0) {
+ unsigned int MaxSteps = 0,
+ bool TopologicalPrune = false) {
+ SmallVector<const SDNode *, 8> DeferredNodes;
if (Visited.count(N))
return true;
+
+ // Node Id's are assigned in three places: As a topological
+ // ordering (> 0), during legalization (results in values set to
+ // 0), new nodes (set to -1). If N has a topolgical id then we
+ // know that all nodes with ids smaller than it cannot be
+ // successors and we need not check them. Filter out all node
+ // that can't be matches. We add them to the worklist before exit
+ // in case of multiple calls. Note that during selection the topological id
+ // may be violated if a node's predecessor is selected before it. We mark
+ // this at selection negating the id of unselected successors and
+ // restricting topological pruning to positive ids.
+
+ int NId = N->getNodeId();
+ // If we Invalidated the Id, reconstruct original NId.
+ if (NId < -1)
+ NId = -(NId + 1);
+
+ bool Found = false;
while (!Worklist.empty()) {
const SDNode *M = Worklist.pop_back_val();
- bool Found = false;
+ int MId = M->getNodeId();
+ if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
+ (MId > 0) && (MId < NId)) {
+ DeferredNodes.push_back(M);
+ continue;
+ }
for (const SDValue &OpV : M->op_values()) {
SDNode *Op = OpV.getNode();
if (Visited.insert(Op).second)
@@ -814,11 +873,16 @@ public:
Found = true;
}
if (Found)
- return true;
+ break;
if (MaxSteps != 0 && Visited.size() >= MaxSteps)
- return false;
+ break;
}
- return false;
+ // Push deferred nodes back on worklist.
+ Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
+ // If we bailed early, conservatively return found.
+ if (MaxSteps != 0 && Visited.size() >= MaxSteps)
+ return true;
+ return Found;
}
/// Return true if all the users of N are contained in Nodes.
@@ -884,6 +948,7 @@ public:
const SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
+ bool isFast() { return Flags.isFast(); }
/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
@@ -1089,10 +1154,18 @@ inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
}
+inline void SDValue::dump() const {
+ return Node->dump();
+}
+
inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
}
+inline void SDValue::dumpr() const {
+ return Node->dumpr();
+}
+
inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
}
@@ -1173,7 +1246,7 @@ protected:
public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
- EVT MemoryVT, MachineMemOperand *MMO);
+ EVT memvt, MachineMemOperand *MMO);
bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }
@@ -1190,7 +1263,8 @@ public:
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
- /// nodes with debug info with nodes without debug info.
+ /// nodes with debug info with nodes without debug info. Same is about
+ /// isDivergent bit.
unsigned getRawSubclassData() const {
uint16_t Data;
union {
@@ -1199,6 +1273,7 @@ public:
};
memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
SDNodeBits.HasDebugValue = 0;
+ SDNodeBits.IsDivergent = false;
memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
return Data;
}
@@ -1267,6 +1342,7 @@ public:
N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
@@ -1318,6 +1394,7 @@ public:
N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
@@ -1421,9 +1498,8 @@ class ConstantSDNode : public SDNode {
const ConstantInt *Value;
- ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val,
- const DebugLoc &DL, EVT VT)
- : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL,
+ ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
+ : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
getSDVTList(VT)),
Value(val) {
ConstantSDNodeBits.IsOpaque = isOpaque;
@@ -1459,10 +1535,9 @@ class ConstantFPSDNode : public SDNode {
const ConstantFP *Value;
- ConstantFPSDNode(bool isTarget, const ConstantFP *val, const DebugLoc &DL,
- EVT VT)
- : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, DL,
- getSDVTList(VT)),
+ ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
+ : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
+ DebugLoc(), getSDVTList(VT)),
Value(val) {}
public:
@@ -1519,10 +1594,10 @@ bool isOneConstant(SDValue V);
bool isBitwiseNot(SDValue V);
/// Returns the SDNode if it is a constant splat BuildVector or constant int.
-ConstantSDNode *isConstOrConstSplat(SDValue V);
+ConstantSDNode *isConstOrConstSplat(SDValue N);
/// Returns the SDNode if it is a constant splat BuildVector or constant float.
-ConstantFPSDNode *isConstOrConstSplatFP(SDValue V);
+ConstantFPSDNode *isConstOrConstSplatFP(SDValue N);
class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;
@@ -1533,7 +1608,7 @@ class GlobalAddressSDNode : public SDNode {
GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
const GlobalValue *GA, EVT VT, int64_t o,
- unsigned char TargetFlags);
+ unsigned char TF);
public:
const GlobalValue *getGlobal() const { return TheGlobal; }
@@ -1714,13 +1789,13 @@ public:
unsigned MinSplatBits = 0,
bool isBigEndian = false) const;
- /// \brief Returns the splatted value or a null value if this is not a splat.
+ /// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
- /// \brief Returns the splatted constant or null if this is not a constant
+ /// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
@@ -1728,7 +1803,7 @@ public:
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;
- /// \brief Returns the splatted constant FP or null if this is not a constant
+ /// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
@@ -1736,7 +1811,7 @@ public:
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
- /// \brief If this is a constant FP splat and the splatted constant FP is an
+ /// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value. Otherwise,
/// return -1.
///
@@ -2120,13 +2195,14 @@ public:
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
// In the both nodes address is Op1, mask is Op2:
- // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value
- // MaskedScatterSDNode (Chain, value, mask, base, index)
+ // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
+ // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex() const { return getOperand(4); }
const SDValue &getMask() const { return getOperand(2); }
const SDValue &getValue() const { return getOperand(1); }
+ const SDValue &getScale() const { return getOperand(5); }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MGATHER ||
@@ -2329,6 +2405,17 @@ namespace ISD {
cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}
+ /// Attempt to match a unary predicate against a scalar/splat constant or
+ /// every element of a constant BUILD_VECTOR.
+ bool matchUnaryPredicate(SDValue Op,
+ std::function<bool(ConstantSDNode *)> Match);
+
+ /// Attempt to match a binary predicate against a pair of scalar/splat
+ /// constants or every element of a pair of constant BUILD_VECTORs.
+ bool matchBinaryPredicate(
+ SDValue LHS, SDValue RHS,
+ std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match);
+
} // end namespace ISD
} // end namespace llvm
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-09 04:24:30 +0000