diff options
Diffstat (limited to 'include/llvm/CodeGen/SelectionDAGNodes.h')
| -rw-r--r-- | include/llvm/CodeGen/SelectionDAGNodes.h | 185 |
1 files changed, 154 insertions, 31 deletions
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 10f284179084..5aab9643e09d 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,6 +183,7 @@ public: inline unsigned getNumOperands() const; inline const SDValue &getOperand(unsigned i) const; inline uint64_t getConstantOperandVal(unsigned i) const; + inline const APInt &getConstantOperandAPInt(unsigned i) const; inline bool isTargetMemoryOpcode() const; inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; @@ -232,7 +232,6 @@ template<> struct DenseMapInfo<SDValue> { return LHS == RHS; } }; -template <> struct isPodLike<SDValue> { static const bool value = true; }; /// Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. @@ -369,6 +368,13 @@ private: bool ApproximateFuncs : 1; bool AllowReassociation : 1; + // We assume instructions do not raise floating-point exceptions by default, + // and only those marked explicitly may do so. We could choose to represent + // this via a positive "FPExcept" flags like on the MI level, but having a + // negative "NoFPExcept" flag here (that defaults to true) makes the flag + // intersection logic more straightforward. + bool NoFPExcept : 1; + public: /// Default constructor turns off all optimization flags. SDNodeFlags() @@ -376,7 +382,7 @@ public: Exact(false), NoNaNs(false), NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), AllowContract(false), ApproximateFuncs(false), - AllowReassociation(false) {} + AllowReassociation(false), NoFPExcept(true) {} /// Propagate the fast-math-flags from an IR FPMathOperator. void copyFMF(const FPMathOperator &FPMO) { @@ -439,6 +445,10 @@ public: setDefined(); AllowReassociation = b; } + void setFPExcept(bool b) { + setDefined(); + NoFPExcept = !b; + } // These are accessors for each flag. bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } @@ -452,9 +462,10 @@ public: bool hasAllowContract() const { return AllowContract; } bool hasApproximateFuncs() const { return ApproximateFuncs; } bool hasAllowReassociation() const { return AllowReassociation; } + bool hasFPExcept() const { return !NoFPExcept; } bool isFast() const { - return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && + return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept && AllowContract && ApproximateFuncs && AllowReassociation; } @@ -474,6 +485,7 @@ public: AllowContract &= Flags.AllowContract; ApproximateFuncs &= Flags.ApproximateFuncs; AllowReassociation &= Flags.AllowReassociation; + NoFPExcept &= Flags.NoFPExcept; } }; @@ -489,6 +501,17 @@ protected: // SubclassData. These are designed to fit within a uint16_t so they pack // with NodeType. +#if defined(_AIX) && (!defined(__GNUC__) || defined(__ibmxl__)) +// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words +// and give the `pack` pragma push semantics. +#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)") +#define END_TWO_BYTE_PACK() _Pragma("pack(pop)") +#else +#define BEGIN_TWO_BYTE_PACK() +#define END_TWO_BYTE_PACK() +#endif + +BEGIN_TWO_BYTE_PACK() class SDNodeBitfields { friend class SDNode; friend class MemIntrinsicSDNode; @@ -561,6 +584,9 @@ protected: LoadSDNodeBitfields LoadSDNodeBits; StoreSDNodeBitfields StoreSDNodeBits; }; +END_TWO_BYTE_PACK() +#undef BEGIN_TWO_BYTE_PACK +#undef END_TWO_BYTE_PACK // RawSDNodeBits must cover the entirety of the union. This means that all of // the union's members must have size <= RawSDNodeBits. We write the RHS as @@ -678,6 +704,8 @@ public: case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_ROUND: + case ISD::STRICT_FP_EXTEND: return true; } } @@ -898,9 +926,17 @@ public: /// Return the number of values used by this operation. unsigned getNumOperands() const { return NumOperands; } + /// Return the maximum number of operands that a SDNode can hold. + static constexpr size_t getMaxNumOperands() { + return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); + } + /// Helper method returns the integer value of a ConstantSDNode operand. inline uint64_t getConstantOperandVal(unsigned Num) const; + /// Helper method returns the APInt of a ConstantSDNode operand. + inline const APInt &getConstantOperandAPInt(unsigned Num) const; + const SDValue &getOperand(unsigned Num) const { assert(Num < NumOperands && "Invalid child # of SDNode!"); return OperandList[Num]; @@ -1128,6 +1164,10 @@ inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { return Node->getConstantOperandVal(i); } +inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { + return Node->getConstantOperandAPInt(i); +} + inline bool SDValue::isTargetOpcode() const { return Node->isTargetOpcode(); } @@ -1356,6 +1396,8 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD_FADD || + N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || N->getOpcode() == ISD::ATOMIC_LOAD || N->getOpcode() == ISD::ATOMIC_STORE || N->getOpcode() == ISD::MLOAD || @@ -1372,7 +1414,10 @@ class AtomicSDNode : public MemSDNode { public: AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {} + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || + MMO->isAtomic()) && "then why are we using an AtomicSDNode?"); + } const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } @@ -1408,6 +1453,8 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD_FADD || + N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || N->getOpcode() == ISD::ATOMIC_LOAD || N->getOpcode() == ISD::ATOMIC_STORE; } @@ -1467,14 +1514,16 @@ public: bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } - int getSplatIndex() const { + int getSplatIndex() const { assert(isSplat() && "Cannot get splat index for non-splat!"); EVT VT = getValueType(0); - for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) if (Mask[i] >= 0) return Mask[i]; - } - llvm_unreachable("Splat with all undef indices?"); + + // We can choose any index value here and be correct because all elements + // are undefined. Return 0 for better potential for callers to simplify. + return 0; } static bool isSplatMask(const int *Mask, EVT VT); @@ -1536,6 +1585,10 @@ uint64_t SDNode::getConstantOperandVal(unsigned Num) const { return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); } +const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { + return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); +} + class ConstantFPSDNode : public SDNode { friend class SelectionDAG; @@ -1603,20 +1656,36 @@ SDValue peekThroughBitcasts(SDValue V); /// If \p V is not a bitcasted one-use value, it is returned as-is. SDValue peekThroughOneUseBitcasts(SDValue V); +/// Return the non-extracted vector source operand of \p V if it exists. +/// If \p V is not an extracted subvector, it is returned as-is. +SDValue peekThroughExtractSubvectors(SDValue V); + /// Returns true if \p V is a bitwise not operation. Assumes that an all ones /// constant is canonicalized to be operand 1. -bool isBitwiseNot(SDValue V); +bool isBitwiseNot(SDValue V, bool AllowUndefs = false); /// Returns the SDNode if it is a constant splat BuildVector or constant int. -ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false); +ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, + bool AllowTruncation = false); + +/// Returns the SDNode if it is a demanded constant splat BuildVector or +/// constant int. +ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, + bool AllowUndefs = false, + bool AllowTruncation = false); /// Returns the SDNode if it is a constant splat BuildVector or constant float. ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); +/// Returns the SDNode if it is a demanded constant splat BuildVector or +/// constant float. +ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, + bool AllowUndefs = false); + /// Return true if the value is a constant 0 integer or a splatted vector of -/// a constant 0 integer (with no undefs). +/// a constant 0 integer (with no undefs by default). /// Build vector implicit truncation is not an issue for null values. -bool isNullOrNullSplat(SDValue V); +bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); /// Return true if the value is a constant 1 integer or a splatted vector of a /// constant 1 integer (with no undefs). @@ -1673,6 +1742,38 @@ public: } }; +/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate +/// the offet and size that are started/ended in the underlying FrameIndex. +class LifetimeSDNode : public SDNode { + friend class SelectionDAG; + int64_t Size; + int64_t Offset; // -1 if offset is unknown. + + LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, + SDVTList VTs, int64_t Size, int64_t Offset) + : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} +public: + int64_t getFrameIndex() const { + return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); + } + + bool hasOffset() const { return Offset >= 0; } + int64_t getOffset() const { + assert(hasOffset() && "offset is unknown"); + return Offset; + } + int64_t getSize() const { + assert(hasOffset() && "offset is unknown"); + return Size; + } + + // Methods to support isa and dyn_cast + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::LIFETIME_START || + N->getOpcode() == ISD::LIFETIME_END; + } +}; + class JumpTableSDNode : public SDNode { friend class SelectionDAG; @@ -1818,12 +1919,31 @@ public: unsigned MinSplatBits = 0, bool isBigEndian = false) const; + /// Returns the demanded splatted value or a null value if this is not a + /// splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + SDValue getSplatValue(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted value or a null value if this is not a splat. /// /// If passed a non-null UndefElements bitvector, it will resize it to match /// the vector width and set the bits where elements are undef. SDValue getSplatValue(BitVector *UndefElements = nullptr) const; + /// Returns the demanded splatted constant or null if this is not a constant + /// splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantSDNode * + getConstantSplatNode(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted constant or null if this is not a constant /// splat. /// @@ -1832,6 +1952,16 @@ public: ConstantSDNode * getConstantSplatNode(BitVector *UndefElements = nullptr) const; + /// Returns the demanded splatted constant FP or null if this is not a + /// constant FP splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantFPSDNode * + getConstantFPSplatNode(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted constant FP or null if this is not a constant /// FP splat. /// @@ -1956,8 +2086,10 @@ class LabelSDNode : public SDNode { MCSymbol *Label; - LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L) - : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} + LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) + : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { + assert(LabelSDNode::classof(this) && "not a label opcode"); + } public: MCSymbol *getLabel() const { return Label; } @@ -2049,6 +2181,8 @@ public: : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { LSBaseSDNodeBits.AddressingMode = AM; assert(getAddressingMode() == AM && "Value truncated"); + assert((!MMO->isAtomic() || MMO->isVolatile()) && + "use an AtomicSDNode instead for non-volatile atomics"); } const SDValue &getOffset() const { @@ -2473,18 +2607,6 @@ namespace ISD { cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; } - /// Return true if the node is a math/logic binary operator. This corresponds - /// to the IR function of the same name. - inline bool isBinaryOp(const SDNode *N) { - auto Op = N->getOpcode(); - return (Op == ISD::ADD || Op == ISD::SUB || Op == ISD::MUL || - Op == ISD::AND || Op == ISD::OR || Op == ISD::XOR || - Op == ISD::SHL || Op == ISD::SRL || Op == ISD::SRA || - Op == ISD::SDIV || Op == ISD::UDIV || Op == ISD::SREM || - Op == ISD::UREM || Op == ISD::FADD || Op == ISD::FSUB || - Op == ISD::FMUL || Op == ISD::FDIV || Op == ISD::FREM); - } - /// Attempt to match a unary predicate against a scalar/splat constant or /// every element of a constant BUILD_VECTOR. /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. @@ -2495,10 +2617,11 @@ namespace ISD { /// Attempt to match a binary predicate against a pair of scalar/splat /// constants or every element of a pair of constant BUILD_VECTORs. /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. + /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. bool matchBinaryPredicate( SDValue LHS, SDValue RHS, std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, - bool AllowUndefs = false); + bool AllowUndefs = false, bool AllowTypeMismatch = false); } // end namespace ISD } // end namespace llvm |