diff options
Diffstat (limited to 'contrib/llvm/include/llvm')
546 files changed, 23000 insertions, 9918 deletions
diff --git a/contrib/llvm/include/llvm/ADT/APFloat.h b/contrib/llvm/include/llvm/ADT/APFloat.h index 9c5e392c4808..6c0b6ae78ae3 100644 --- a/contrib/llvm/include/llvm/ADT/APFloat.h +++ b/contrib/llvm/include/llvm/ADT/APFloat.h @@ -1119,6 +1119,21 @@ public: llvm_unreachable("Unexpected semantics"); } + /// We don't rely on operator== working on double values, as + /// it returns true for things that are clearly not equal, like -0.0 and 0.0. + /// As such, this method can be used to do an exact bit-for-bit comparison of + /// two floating point values. + /// + /// We leave the version with the double argument here because it's just so + /// convenient to write "2.0" and the like. Without this function we'd + /// have to duplicate its logic everywhere it's called. + bool isExactlyValue(double V) const { + bool ignored; + APFloat Tmp(V); + Tmp.convert(getSemantics(), APFloat::rmNearestTiesToEven, &ignored); + return bitwiseIsEqual(Tmp); + } + unsigned int convertToHexString(char *DST, unsigned int HexDigits, bool UpperCase, roundingMode RM) const { APFLOAT_DISPATCH_ON_SEMANTICS( diff --git a/contrib/llvm/include/llvm/ADT/APInt.h b/contrib/llvm/include/llvm/ADT/APInt.h index a1cce6e5fe17..c81363cc16b7 100644 --- a/contrib/llvm/include/llvm/ADT/APInt.h +++ b/contrib/llvm/include/llvm/ADT/APInt.h @@ -1724,13 +1724,13 @@ public: /// @{ /// \returns the floor log base 2 of this APInt. - unsigned logBase2() const { return BitWidth - 1 - countLeadingZeros(); } + unsigned logBase2() const { return getActiveBits() - 1; } /// \returns the ceil log base 2 of this APInt. unsigned ceilLogBase2() const { APInt temp(*this); --temp; - return BitWidth - temp.countLeadingZeros(); + return temp.getActiveBits(); } /// \returns the nearest log base 2 of this APInt. Ties round up. diff --git a/contrib/llvm/include/llvm/ADT/ArrayRef.h b/contrib/llvm/include/llvm/ADT/ArrayRef.h index 925ebafc3fed..5f7a769ddac4 100644 --- a/contrib/llvm/include/llvm/ADT/ArrayRef.h +++ b/contrib/llvm/include/llvm/ADT/ArrayRef.h @@ -294,7 +294,7 @@ namespace llvm { using reverse_iterator = std::reverse_iterator<iterator>; /// Construct an empty MutableArrayRef. - /*implicit*/ MutableArrayRef() : ArrayRef<T>() {} + /*implicit*/ MutableArrayRef() = default; /// Construct an empty MutableArrayRef from None. /*implicit*/ MutableArrayRef(NoneType) : ArrayRef<T>() {} diff --git a/contrib/llvm/include/llvm/ADT/BitVector.h b/contrib/llvm/include/llvm/ADT/BitVector.h index e68ef5f53d10..99147fec4d4c 100644 --- a/contrib/llvm/include/llvm/ADT/BitVector.h +++ b/contrib/llvm/include/llvm/ADT/BitVector.h @@ -911,7 +911,7 @@ public: size_t getBitCapacity() const { return Bits.size() * BITWORD_SIZE; } }; -static inline size_t capacity_in_bytes(const BitVector &X) { +inline size_t capacity_in_bytes(const BitVector &X) { return X.getMemorySize(); } diff --git a/contrib/llvm/include/llvm/ADT/DenseMap.h b/contrib/llvm/include/llvm/ADT/DenseMap.h index b311e69ec9d3..ba60b7972a8f 100644 --- a/contrib/llvm/include/llvm/ADT/DenseMap.h +++ b/contrib/llvm/include/llvm/ADT/DenseMap.h @@ -19,6 +19,7 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/ReverseIteration.h" #include "llvm/Support/type_traits.h" #include <algorithm> #include <cassert> @@ -67,18 +68,26 @@ public: DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>; inline iterator begin() { - // When the map is empty, avoid the overhead of AdvancePastEmptyBuckets(). - return empty() ? end() : iterator(getBuckets(), getBucketsEnd(), *this); + // When the map is empty, avoid the overhead of advancing/retreating past + // empty buckets. + if (empty()) + return end(); + if (shouldReverseIterate<KeyT>()) + return makeIterator(getBucketsEnd() - 1, getBuckets(), *this); + return makeIterator(getBuckets(), getBucketsEnd(), *this); } inline iterator end() { - return iterator(getBucketsEnd(), getBucketsEnd(), *this, true); + return makeIterator(getBucketsEnd(), getBucketsEnd(), *this, true); } inline const_iterator begin() const { - return empty() ? end() - : const_iterator(getBuckets(), getBucketsEnd(), *this); + if (empty()) + return end(); + if (shouldReverseIterate<KeyT>()) + return makeConstIterator(getBucketsEnd() - 1, getBuckets(), *this); + return makeConstIterator(getBuckets(), getBucketsEnd(), *this); } inline const_iterator end() const { - return const_iterator(getBucketsEnd(), getBucketsEnd(), *this, true); + return makeConstIterator(getBucketsEnd(), getBucketsEnd(), *this, true); } LLVM_NODISCARD bool empty() const { @@ -107,17 +116,23 @@ public: } const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); - unsigned NumEntries = getNumEntries(); - for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { - if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { - if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { - P->getSecond().~ValueT(); - --NumEntries; - } + if (isPodLike<KeyT>::value && isPodLike<ValueT>::value) { + // Use a simpler loop when these are trivial types. + for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) P->getFirst() = EmptyKey; + } else { + unsigned NumEntries = getNumEntries(); + for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { + if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { + P->getSecond().~ValueT(); + --NumEntries; + } + P->getFirst() = EmptyKey; + } } + assert(NumEntries == 0 && "Node count imbalance!"); } - assert(NumEntries == 0 && "Node count imbalance!"); setNumEntries(0); setNumTombstones(0); } @@ -131,13 +146,13 @@ public: iterator find(const_arg_type_t<KeyT> Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return iterator(TheBucket, getBucketsEnd(), *this, true); + return makeIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } const_iterator find(const_arg_type_t<KeyT> Val) const { const BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return const_iterator(TheBucket, getBucketsEnd(), *this, true); + return makeConstIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } @@ -150,14 +165,14 @@ public: iterator find_as(const LookupKeyT &Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return iterator(TheBucket, getBucketsEnd(), *this, true); + return makeIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } template<class LookupKeyT> const_iterator find_as(const LookupKeyT &Val) const { const BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return const_iterator(TheBucket, getBucketsEnd(), *this, true); + return makeConstIterator(TheBucket, getBucketsEnd(), *this, true); return end(); } @@ -191,14 +206,16 @@ public: std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&... Args) { BucketT *TheBucket; if (LookupBucketFor(Key, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucket(TheBucket, std::move(Key), std::forward<Ts>(Args)...); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } // Inserts key,value pair into the map if the key isn't already in the map. @@ -208,13 +225,15 @@ public: std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&... Args) { BucketT *TheBucket; if (LookupBucketFor(Key, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucket(TheBucket, Key, std::forward<Ts>(Args)...); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } /// Alternate version of insert() which allows a different, and possibly @@ -227,14 +246,16 @@ public: const LookupKeyT &Val) { BucketT *TheBucket; if (LookupBucketFor(Val, TheBucket)) - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - false); // Already in map. + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + false); // Already in map. // Otherwise, insert the new element. TheBucket = InsertIntoBucketWithLookup(TheBucket, std::move(KV.first), std::move(KV.second), Val); - return std::make_pair(iterator(TheBucket, getBucketsEnd(), *this, true), - true); + return std::make_pair( + makeIterator(TheBucket, getBucketsEnd(), *this, true), + true); } /// insert - Range insertion of pairs. @@ -405,6 +426,26 @@ protected: } private: + iterator makeIterator(BucketT *P, BucketT *E, + DebugEpochBase &Epoch, + bool NoAdvance=false) { + if (shouldReverseIterate<KeyT>()) { + BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; + return iterator(B, E, Epoch, NoAdvance); + } + return iterator(P, E, Epoch, NoAdvance); + } + + const_iterator makeConstIterator(const BucketT *P, const BucketT *E, + const DebugEpochBase &Epoch, + const bool NoAdvance=false) const { + if (shouldReverseIterate<KeyT>()) { + const BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; + return const_iterator(B, E, Epoch, NoAdvance); + } + return const_iterator(P, E, Epoch, NoAdvance); + } + unsigned getNumEntries() const { return static_cast<const DerivedT *>(this)->getNumEntries(); } @@ -1089,7 +1130,13 @@ public: bool NoAdvance = false) : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E) { assert(isHandleInSync() && "invalid construction!"); - if (!NoAdvance) AdvancePastEmptyBuckets(); + + if (NoAdvance) return; + if (shouldReverseIterate<KeyT>()) { + RetreatPastEmptyBuckets(); + return; + } + AdvancePastEmptyBuckets(); } // Converting ctor from non-const iterators to const iterators. SFINAE'd out @@ -1103,10 +1150,14 @@ public: reference operator*() const { assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate<KeyT>()) + return Ptr[-1]; return *Ptr; } pointer operator->() const { assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate<KeyT>()) + return &(Ptr[-1]); return Ptr; } @@ -1127,6 +1178,11 @@ public: inline DenseMapIterator& operator++() { // Preincrement assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate<KeyT>()) { + --Ptr; + RetreatPastEmptyBuckets(); + return *this; + } ++Ptr; AdvancePastEmptyBuckets(); return *this; @@ -1138,6 +1194,7 @@ public: private: void AdvancePastEmptyBuckets() { + assert(Ptr <= End); const KeyT Empty = KeyInfoT::getEmptyKey(); const KeyT Tombstone = KeyInfoT::getTombstoneKey(); @@ -1145,11 +1202,20 @@ private: KeyInfoT::isEqual(Ptr->getFirst(), Tombstone))) ++Ptr; } + + void RetreatPastEmptyBuckets() { + assert(Ptr >= End); + const KeyT Empty = KeyInfoT::getEmptyKey(); + const KeyT Tombstone = KeyInfoT::getTombstoneKey(); + + while (Ptr != End && (KeyInfoT::isEqual(Ptr[-1].getFirst(), Empty) || + KeyInfoT::isEqual(Ptr[-1].getFirst(), Tombstone))) + --Ptr; + } }; -template<typename KeyT, typename ValueT, typename KeyInfoT> -static inline size_t -capacity_in_bytes(const DenseMap<KeyT, ValueT, KeyInfoT> &X) { +template <typename KeyT, typename ValueT, typename KeyInfoT> +inline size_t capacity_in_bytes(const DenseMap<KeyT, ValueT, KeyInfoT> &X) { return X.getMemorySize(); } diff --git a/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h b/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h index af293d4c1422..e3f48433c69f 100644 --- a/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h +++ b/contrib/llvm/include/llvm/ADT/EquivalenceClasses.h @@ -239,6 +239,16 @@ public: return L1; } + // isEquivalent - Return true if V1 is equivalent to V2. This can happen if + // V1 is equal to V2 or if they belong to one equivalence class. + bool isEquivalent(const ElemTy &V1, const ElemTy &V2) const { + // Fast path: any element is equivalent to itself. + if (V1 == V2) + return true; + auto It = findLeader(V1); + return It != member_end() && It == findLeader(V2); + } + class member_iterator : public std::iterator<std::forward_iterator_tag, const ElemTy, ptrdiff_t> { friend class EquivalenceClasses; diff --git a/contrib/llvm/include/llvm/ADT/FoldingSet.h b/contrib/llvm/include/llvm/ADT/FoldingSet.h index c5987a947e18..e363e69d032a 100644 --- a/contrib/llvm/include/llvm/ADT/FoldingSet.h +++ b/contrib/llvm/include/llvm/ADT/FoldingSet.h @@ -1,4 +1,4 @@ -//===-- llvm/ADT/FoldingSet.h - Uniquing Hash Set ---------------*- C++ -*-===// +//===- llvm/ADT/FoldingSet.h - Uniquing Hash Set ----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -115,11 +115,9 @@ class FoldingSetBase { protected: /// Buckets - Array of bucket chains. - /// void **Buckets; /// NumBuckets - Length of the Buckets array. Always a power of 2. - /// unsigned NumBuckets; /// NumNodes - Number of nodes in the folding set. Growth occurs when NumNodes @@ -135,14 +133,13 @@ public: //===--------------------------------------------------------------------===// /// Node - This class is used to maintain the singly linked bucket list in /// a folding set. - /// class Node { private: // NextInFoldingSetBucket - next link in the bucket list. - void *NextInFoldingSetBucket; + void *NextInFoldingSetBucket = nullptr; public: - Node() : NextInFoldingSetBucket(nullptr) {} + Node() = default; // Accessors void *getNextInBucket() const { return NextInFoldingSetBucket; } @@ -221,7 +218,6 @@ protected: /// DefaultFoldingSetTrait - This class provides default implementations /// for FoldingSetTrait implementations. -/// template<typename T> struct DefaultFoldingSetTrait { static void Profile(const T &X, FoldingSetNodeID &ID) { X.Profile(ID); @@ -307,7 +303,6 @@ public: /// FoldingSetNodeID - This class is used to gather all the unique data bits of /// a node. When all the bits are gathered this class is used to produce a /// hash value for the node. -/// class FoldingSetNodeID { /// Bits - Vector of all the data bits that make the node unique. /// Use a SmallVector to avoid a heap allocation in the common case. @@ -320,7 +315,6 @@ public: : Bits(Ref.getData(), Ref.getData() + Ref.getSize()) {} /// Add* - Add various data types to Bit data. - /// void AddPointer(const void *Ptr); void AddInteger(signed I); void AddInteger(unsigned I); @@ -344,7 +338,6 @@ public: unsigned ComputeHash() const; /// operator== - Used to compare two nodes to each other. - /// bool operator==(const FoldingSetNodeID &RHS) const; bool operator==(const FoldingSetNodeIDRef RHS) const; @@ -363,7 +356,7 @@ public: }; // Convenience type to hide the implementation of the folding set. -typedef FoldingSetBase::Node FoldingSetNode; +using FoldingSetNode = FoldingSetBase::Node; template<class T> class FoldingSetIterator; template<class T> class FoldingSetBucketIterator; @@ -415,15 +408,17 @@ protected: ~FoldingSetImpl() = default; public: - typedef FoldingSetIterator<T> iterator; + using iterator = FoldingSetIterator<T>; + iterator begin() { return iterator(Buckets); } iterator end() { return iterator(Buckets+NumBuckets); } - typedef FoldingSetIterator<const T> const_iterator; + using const_iterator = FoldingSetIterator<const T>; + const_iterator begin() const { return const_iterator(Buckets); } const_iterator end() const { return const_iterator(Buckets+NumBuckets); } - typedef FoldingSetBucketIterator<T> bucket_iterator; + using bucket_iterator = FoldingSetBucketIterator<T>; bucket_iterator bucket_begin(unsigned hash) { return bucket_iterator(Buckets + (hash & (NumBuckets-1))); @@ -503,9 +498,7 @@ template <class T> class FoldingSet final : public FoldingSetImpl<T> { } public: - explicit FoldingSet(unsigned Log2InitSize = 6) - : Super(Log2InitSize) {} - + explicit FoldingSet(unsigned Log2InitSize = 6) : Super(Log2InitSize) {} FoldingSet(FoldingSet &&Arg) = default; FoldingSet &operator=(FoldingSet &&RHS) = default; }; @@ -552,8 +545,7 @@ class ContextualFoldingSet final : public FoldingSetImpl<T> { public: explicit ContextualFoldingSet(Ctx Context, unsigned Log2InitSize = 6) - : Super(Log2InitSize), Context(Context) - {} + : Super(Log2InitSize), Context(Context) {} Ctx getContext() const { return Context; } }; @@ -569,15 +561,15 @@ class FoldingSetVector { VectorT Vector; public: - explicit FoldingSetVector(unsigned Log2InitSize = 6) - : Set(Log2InitSize) { - } + explicit FoldingSetVector(unsigned Log2InitSize = 6) : Set(Log2InitSize) {} + + using iterator = pointee_iterator<typename VectorT::iterator>; - typedef pointee_iterator<typename VectorT::iterator> iterator; iterator begin() { return Vector.begin(); } iterator end() { return Vector.end(); } - typedef pointee_iterator<typename VectorT::const_iterator> const_iterator; + using const_iterator = pointee_iterator<typename VectorT::const_iterator>; + const_iterator begin() const { return Vector.begin(); } const_iterator end() const { return Vector.end(); } @@ -667,15 +659,13 @@ public: /// FoldingSetBucketIteratorImpl - This is the common bucket iterator support /// shared by all folding sets, which knows how to walk a particular bucket /// of a folding set hash table. - class FoldingSetBucketIteratorImpl { protected: void *Ptr; explicit FoldingSetBucketIteratorImpl(void **Bucket); - FoldingSetBucketIteratorImpl(void **Bucket, bool) - : Ptr(Bucket) {} + FoldingSetBucketIteratorImpl(void **Bucket, bool) : Ptr(Bucket) {} void advance() { void *Probe = static_cast<FoldingSetNode*>(Ptr)->getNextInBucket(); diff --git a/contrib/llvm/include/llvm/ADT/MapVector.h b/contrib/llvm/include/llvm/ADT/MapVector.h index 26a555ee1d3b..3d78f4b203c8 100644 --- a/contrib/llvm/include/llvm/ADT/MapVector.h +++ b/contrib/llvm/include/llvm/ADT/MapVector.h @@ -56,6 +56,13 @@ public: size_type size() const { return Vector.size(); } + /// Grow the MapVector so that it can contain at least \p NumEntries items + /// before resizing again. + void reserve(size_type NumEntries) { + Map.reserve(NumEntries); + Vector.reserve(NumEntries); + } + iterator begin() { return Vector.begin(); } const_iterator begin() const { return Vector.begin(); } iterator end() { return Vector.end(); } diff --git a/contrib/llvm/include/llvm/ADT/Optional.h b/contrib/llvm/include/llvm/ADT/Optional.h index b782d9da17ac..2811d5c1e21b 100644 --- a/contrib/llvm/include/llvm/ADT/Optional.h +++ b/contrib/llvm/include/llvm/ADT/Optional.h @@ -27,8 +27,7 @@ namespace llvm { -template<typename T> -class Optional { +template <typename T> class Optional { AlignedCharArrayUnion<T> storage; bool hasVal = false; @@ -38,18 +37,14 @@ public: Optional(NoneType) {} explicit Optional() {} - Optional(const T &y) : hasVal(true) { - new (storage.buffer) T(y); - } + Optional(const T &y) : hasVal(true) { new (storage.buffer) T(y); } Optional(const Optional &O) : hasVal(O.hasVal) { if (hasVal) new (storage.buffer) T(*O); } - Optional(T &&y) : hasVal(true) { - new (storage.buffer) T(std::forward<T>(y)); - } + Optional(T &&y) : hasVal(true) { new (storage.buffer) T(std::forward<T>(y)); } Optional(Optional<T> &&O) : hasVal(O) { if (O) { @@ -58,9 +53,7 @@ public: } } - ~Optional() { - reset(); - } + ~Optional() { reset(); } Optional &operator=(T &&y) { if (hasVal) @@ -83,14 +76,13 @@ public: } /// Create a new object by constructing it in place with the given arguments. - template<typename ...ArgTypes> - void emplace(ArgTypes &&...Args) { + template <typename... ArgTypes> void emplace(ArgTypes &&... Args) { reset(); hasVal = true; new (storage.buffer) T(std::forward<ArgTypes>(Args)...); } - static inline Optional create(const T* y) { + static inline Optional create(const T *y) { return y ? Optional(*y) : Optional(); } @@ -124,17 +116,35 @@ public: } } - const T* getPointer() const { assert(hasVal); return reinterpret_cast<const T*>(storage.buffer); } - T* getPointer() { assert(hasVal); return reinterpret_cast<T*>(storage.buffer); } - const T& getValue() const LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } - T& getValue() LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } + const T *getPointer() const { + assert(hasVal); + return reinterpret_cast<const T *>(storage.buffer); + } + T *getPointer() { + assert(hasVal); + return reinterpret_cast<T *>(storage.buffer); + } + const T &getValue() const LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } + T &getValue() LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } explicit operator bool() const { return hasVal; } bool hasValue() const { return hasVal; } - const T* operator->() const { return getPointer(); } - T* operator->() { return getPointer(); } - const T& operator*() const LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } - T& operator*() LLVM_LVALUE_FUNCTION { assert(hasVal); return *getPointer(); } + const T *operator->() const { return getPointer(); } + T *operator->() { return getPointer(); } + const T &operator*() const LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } + T &operator*() LLVM_LVALUE_FUNCTION { + assert(hasVal); + return *getPointer(); + } template <typename U> constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION { @@ -142,8 +152,14 @@ public: } #if LLVM_HAS_RVALUE_REFERENCE_THIS - T&& getValue() && { assert(hasVal); return std::move(*getPointer()); } - T&& operator*() && { assert(hasVal); return std::move(*getPointer()); } + T &&getValue() && { + assert(hasVal); + return std::move(*getPointer()); + } + T &&operator*() && { + assert(hasVal); + return std::move(*getPointer()); + } template <typename U> T getValueOr(U &&value) && { diff --git a/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h b/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h index 34323b5b8af4..ab4e1048a5bc 100644 --- a/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h +++ b/contrib/llvm/include/llvm/ADT/PointerEmbeddedInt.h @@ -52,7 +52,7 @@ class PointerEmbeddedInt { explicit RawValueTag() = default; }; - friend class PointerLikeTypeTraits<PointerEmbeddedInt>; + friend struct PointerLikeTypeTraits<PointerEmbeddedInt>; explicit PointerEmbeddedInt(uintptr_t Value, RawValueTag) : Value(Value) {} @@ -80,10 +80,9 @@ public: // Provide pointer like traits to support use with pointer unions and sum // types. template <typename IntT, int Bits> -class PointerLikeTypeTraits<PointerEmbeddedInt<IntT, Bits>> { +struct PointerLikeTypeTraits<PointerEmbeddedInt<IntT, Bits>> { using T = PointerEmbeddedInt<IntT, Bits>; -public: static inline void *getAsVoidPointer(const T &P) { return reinterpret_cast<void *>(P.Value); } diff --git a/contrib/llvm/include/llvm/ADT/PointerIntPair.h b/contrib/llvm/include/llvm/ADT/PointerIntPair.h index 83fbf127e6da..884d05155bff 100644 --- a/contrib/llvm/include/llvm/ADT/PointerIntPair.h +++ b/contrib/llvm/include/llvm/ADT/PointerIntPair.h @@ -14,15 +14,14 @@ #ifndef LLVM_ADT_POINTERINTPAIR_H #define LLVM_ADT_POINTERINTPAIR_H -#include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include <cassert> +#include <cstdint> #include <limits> namespace llvm { template <typename T> struct DenseMapInfo; - template <typename PointerT, unsigned IntBits, typename PtrTraits> struct PointerIntPairInfo; @@ -39,25 +38,24 @@ struct PointerIntPairInfo; /// for something else. For example, this allows: /// PointerIntPair<PointerIntPair<void*, 1, bool>, 1, bool> /// ... and the two bools will land in different bits. -/// template <typename PointerTy, unsigned IntBits, typename IntType = unsigned, typename PtrTraits = PointerLikeTypeTraits<PointerTy>, typename Info = PointerIntPairInfo<PointerTy, IntBits, PtrTraits>> class PointerIntPair { - intptr_t Value; + intptr_t Value = 0; public: - PointerIntPair() : Value(0) {} + constexpr PointerIntPair() = default; + PointerIntPair(PointerTy PtrVal, IntType IntVal) { setPointerAndInt(PtrVal, IntVal); } + explicit PointerIntPair(PointerTy PtrVal) { initWithPointer(PtrVal); } PointerTy getPointer() const { return Info::getPointer(Value); } - IntType getInt() const { - return (IntType)Info::getInt(Value); - } + IntType getInt() const { return (IntType)Info::getInt(Value); } void setPointer(PointerTy PtrVal) { Value = Info::updatePointer(Value, PtrVal); @@ -88,6 +86,7 @@ public: } void *getOpaqueValue() const { return reinterpret_cast<void *>(Value); } + void setFromOpaqueValue(void *Val) { Value = reinterpret_cast<intptr_t>(Val); } @@ -108,14 +107,18 @@ public: bool operator==(const PointerIntPair &RHS) const { return Value == RHS.Value; } + bool operator!=(const PointerIntPair &RHS) const { return Value != RHS.Value; } + bool operator<(const PointerIntPair &RHS) const { return Value < RHS.Value; } bool operator>(const PointerIntPair &RHS) const { return Value > RHS.Value; } + bool operator<=(const PointerIntPair &RHS) const { return Value <= RHS.Value; } + bool operator>=(const PointerIntPair &RHS) const { return Value >= RHS.Value; } @@ -180,44 +183,51 @@ struct isPodLike<PointerIntPair<PointerTy, IntBits, IntType>> { // Provide specialization of DenseMapInfo for PointerIntPair. template <typename PointerTy, unsigned IntBits, typename IntType> struct DenseMapInfo<PointerIntPair<PointerTy, IntBits, IntType>> { - typedef PointerIntPair<PointerTy, IntBits, IntType> Ty; + using Ty = PointerIntPair<PointerTy, IntBits, IntType>; + static Ty getEmptyKey() { uintptr_t Val = static_cast<uintptr_t>(-1); Val <<= PointerLikeTypeTraits<Ty>::NumLowBitsAvailable; return Ty::getFromOpaqueValue(reinterpret_cast<void *>(Val)); } + static Ty getTombstoneKey() { uintptr_t Val = static_cast<uintptr_t>(-2); Val <<= PointerLikeTypeTraits<PointerTy>::NumLowBitsAvailable; return Ty::getFromOpaqueValue(reinterpret_cast<void *>(Val)); } + static unsigned getHashValue(Ty V) { uintptr_t IV = reinterpret_cast<uintptr_t>(V.getOpaqueValue()); return unsigned(IV) ^ unsigned(IV >> 9); } + static bool isEqual(const Ty &LHS, const Ty &RHS) { return LHS == RHS; } }; // Teach SmallPtrSet that PointerIntPair is "basically a pointer". template <typename PointerTy, unsigned IntBits, typename IntType, typename PtrTraits> -class PointerLikeTypeTraits< +struct PointerLikeTypeTraits< PointerIntPair<PointerTy, IntBits, IntType, PtrTraits>> { -public: static inline void * getAsVoidPointer(const PointerIntPair<PointerTy, IntBits, IntType> &P) { return P.getOpaqueValue(); } + static inline PointerIntPair<PointerTy, IntBits, IntType> getFromVoidPointer(void *P) { return PointerIntPair<PointerTy, IntBits, IntType>::getFromOpaqueValue(P); } + static inline PointerIntPair<PointerTy, IntBits, IntType> getFromVoidPointer(const void *P) { return PointerIntPair<PointerTy, IntBits, IntType>::getFromOpaqueValue(P); } + enum { NumLowBitsAvailable = PtrTraits::NumLowBitsAvailable - IntBits }; }; } // end namespace llvm -#endif + +#endif // LLVM_ADT_POINTERINTPAIR_H diff --git a/contrib/llvm/include/llvm/ADT/PointerSumType.h b/contrib/llvm/include/llvm/ADT/PointerSumType.h index 062544eedf84..e37957160d98 100644 --- a/contrib/llvm/include/llvm/ADT/PointerSumType.h +++ b/contrib/llvm/include/llvm/ADT/PointerSumType.h @@ -11,8 +11,10 @@ #define LLVM_ADT_POINTERSUMTYPE_H #include "llvm/ADT/DenseMapInfo.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include <cassert> +#include <cstdint> +#include <type_traits> namespace llvm { @@ -24,16 +26,15 @@ template <uintptr_t N, typename PointerArgT, typename TraitsArgT = PointerLikeTypeTraits<PointerArgT>> struct PointerSumTypeMember { enum { Tag = N }; - typedef PointerArgT PointerT; - typedef TraitsArgT TraitsT; + using PointerT = PointerArgT; + using TraitsT = TraitsArgT; }; namespace detail { -template <typename TagT, typename... MemberTs> -struct PointerSumTypeHelper; +template <typename TagT, typename... MemberTs> struct PointerSumTypeHelper; -} +} // end namespace detail /// A sum type over pointer-like types. /// @@ -60,12 +61,12 @@ struct PointerSumTypeHelper; /// There is no support for constructing or accessing with a dynamic tag as /// that would fundamentally violate the type safety provided by the sum type. template <typename TagT, typename... MemberTs> class PointerSumType { - uintptr_t Value; + uintptr_t Value = 0; - typedef detail::PointerSumTypeHelper<TagT, MemberTs...> HelperT; + using HelperT = detail::PointerSumTypeHelper<TagT, MemberTs...>; public: - PointerSumType() : Value(0) {} + constexpr PointerSumType() = default; /// A typed constructor for a specific tagged member of the sum type. template <TagT N> @@ -128,14 +129,14 @@ struct PointerSumTypeHelper : MemberTs... { template <TagT N> static void LookupOverload(...); template <TagT N> struct Lookup { // Compute a particular member type by resolving the lookup helper ovorload. - typedef decltype(LookupOverload<N>( - static_cast<PointerSumTypeHelper *>(nullptr))) MemberT; + using MemberT = decltype( + LookupOverload<N>(static_cast<PointerSumTypeHelper *>(nullptr))); /// The Nth member's pointer type. - typedef typename MemberT::PointerT PointerT; + using PointerT = typename MemberT::PointerT; /// The Nth member's traits type. - typedef typename MemberT::TraitsT TraitsT; + using TraitsT = typename MemberT::TraitsT; }; // Next we need to compute the number of bits available for the discriminant @@ -171,35 +172,36 @@ struct PointerSumTypeHelper : MemberTs... { "Each member must pass the checker."); }; -} +} // end namespace detail // Teach DenseMap how to use PointerSumTypes as keys. template <typename TagT, typename... MemberTs> struct DenseMapInfo<PointerSumType<TagT, MemberTs...>> { - typedef PointerSumType<TagT, MemberTs...> SumType; - - typedef detail::PointerSumTypeHelper<TagT, MemberTs...> HelperT; + using SumType = PointerSumType<TagT, MemberTs...>; + using HelperT = detail::PointerSumTypeHelper<TagT, MemberTs...>; enum { SomeTag = HelperT::MinTag }; - typedef typename HelperT::template Lookup<HelperT::MinTag>::PointerT - SomePointerT; - typedef DenseMapInfo<SomePointerT> SomePointerInfo; + using SomePointerT = + typename HelperT::template Lookup<HelperT::MinTag>::PointerT; + using SomePointerInfo = DenseMapInfo<SomePointerT>; static inline SumType getEmptyKey() { return SumType::create<SomeTag>(SomePointerInfo::getEmptyKey()); } + static inline SumType getTombstoneKey() { - return SumType::create<SomeTag>( - SomePointerInfo::getTombstoneKey()); + return SumType::create<SomeTag>(SomePointerInfo::getTombstoneKey()); } + static unsigned getHashValue(const SumType &Arg) { uintptr_t OpaqueValue = Arg.getOpaqueValue(); return DenseMapInfo<uintptr_t>::getHashValue(OpaqueValue); } + static bool isEqual(const SumType &LHS, const SumType &RHS) { return LHS == RHS; } }; -} +} // end namespace llvm -#endif +#endif // LLVM_ADT_POINTERSUMTYPE_H diff --git a/contrib/llvm/include/llvm/ADT/PointerUnion.h b/contrib/llvm/include/llvm/ADT/PointerUnion.h index aeab641f5715..4276859e9254 100644 --- a/contrib/llvm/include/llvm/ADT/PointerUnion.h +++ b/contrib/llvm/include/llvm/ADT/PointerUnion.h @@ -25,7 +25,7 @@ namespace llvm { template <typename T> struct PointerUnionTypeSelectorReturn { - typedef T Return; + using Return = T; }; /// Get a type based on whether two types are the same or not. @@ -33,25 +33,25 @@ template <typename T> struct PointerUnionTypeSelectorReturn { /// For: /// /// \code -/// typedef typename PointerUnionTypeSelector<T1, T2, EQ, NE>::Return Ret; +/// using Ret = typename PointerUnionTypeSelector<T1, T2, EQ, NE>::Return; /// \endcode /// /// Ret will be EQ type if T1 is same as T2 or NE type otherwise. template <typename T1, typename T2, typename RET_EQ, typename RET_NE> struct PointerUnionTypeSelector { - typedef typename PointerUnionTypeSelectorReturn<RET_NE>::Return Return; + using Return = typename PointerUnionTypeSelectorReturn<RET_NE>::Return; }; template <typename T, typename RET_EQ, typename RET_NE> struct PointerUnionTypeSelector<T, T, RET_EQ, RET_NE> { - typedef typename PointerUnionTypeSelectorReturn<RET_EQ>::Return Return; + using Return = typename PointerUnionTypeSelectorReturn<RET_EQ>::Return; }; template <typename T1, typename T2, typename RET_EQ, typename RET_NE> struct PointerUnionTypeSelectorReturn< PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>> { - typedef - typename PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>::Return Return; + using Return = + typename PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>::Return; }; /// Provide PointerLikeTypeTraits for void* that is used by PointerUnion @@ -86,8 +86,8 @@ public: /// X = P.get<int*>(); // runtime assertion failure. template <typename PT1, typename PT2> class PointerUnion { public: - typedef PointerIntPair<void *, 1, bool, PointerUnionUIntTraits<PT1, PT2>> - ValTy; + using ValTy = + PointerIntPair<void *, 1, bool, PointerUnionUIntTraits<PT1, PT2>>; private: ValTy Val; @@ -102,7 +102,6 @@ private: public: PointerUnion() = default; - PointerUnion(PT1 V) : Val(const_cast<void *>( PointerLikeTypeTraits<PT1>::getAsVoidPointer(V))) {} @@ -117,14 +116,15 @@ public: // we recursively strip off low bits if we have a nested PointerUnion. return !PointerLikeTypeTraits<PT1>::getFromVoidPointer(Val.getPointer()); } + explicit operator bool() const { return !isNull(); } /// Test if the Union currently holds the type matching T. template <typename T> int is() const { - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, IsPT1, ::llvm::PointerUnionTypeSelector< - PT2, T, IsPT2, UNION_DOESNT_CONTAIN_TYPE<T>>>::Return - Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, IsPT1, + ::llvm::PointerUnionTypeSelector<PT2, T, IsPT2, + UNION_DOESNT_CONTAIN_TYPE<T>>>::Return; int TyNo = Ty::Num; return static_cast<int>(Val.getInt()) == TyNo; } @@ -158,7 +158,8 @@ public: assert( get<PT1>() == Val.getPointer() && "Can't get the address because PointerLikeTypeTraits changes the ptr"); - return const_cast<PT1 *>(reinterpret_cast<const PT1 *>(Val.getAddrOfPointer())); + return const_cast<PT1 *>( + reinterpret_cast<const PT1 *>(Val.getAddrOfPointer())); } /// Assignment from nullptr which just clears the union. @@ -207,8 +208,7 @@ bool operator<(PointerUnion<PT1, PT2> lhs, PointerUnion<PT1, PT2> rhs) { // Teach SmallPtrSet that PointerUnion is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits)-1. template <typename PT1, typename PT2> -class PointerLikeTypeTraits<PointerUnion<PT1, PT2>> { -public: +struct PointerLikeTypeTraits<PointerUnion<PT1, PT2>> { static inline void *getAsVoidPointer(const PointerUnion<PT1, PT2> &P) { return P.getOpaqueValue(); } @@ -228,19 +228,22 @@ public: /// for usage. template <typename PT1, typename PT2, typename PT3> class PointerUnion3 { public: - typedef PointerUnion<PT1, PT2> InnerUnion; - typedef PointerUnion<InnerUnion, PT3> ValTy; + using InnerUnion = PointerUnion<PT1, PT2>; + using ValTy = PointerUnion<InnerUnion, PT3>; private: ValTy Val; struct IsInnerUnion { ValTy Val; + IsInnerUnion(ValTy val) : Val(val) {} + template <typename T> int is() const { return Val.template is<InnerUnion>() && Val.template get<InnerUnion>().template is<T>(); } + template <typename T> T get() const { return Val.template get<InnerUnion>().template get<T>(); } @@ -248,14 +251,15 @@ private: struct IsPT3 { ValTy Val; + IsPT3(ValTy val) : Val(val) {} + template <typename T> int is() const { return Val.template is<T>(); } template <typename T> T get() const { return Val.template get<T>(); } }; public: PointerUnion3() = default; - PointerUnion3(PT1 V) { Val = InnerUnion(V); } PointerUnion3(PT2 V) { Val = InnerUnion(V); } PointerUnion3(PT3 V) { Val = V; } @@ -268,10 +272,9 @@ public: /// Test if the Union currently holds the type matching T. template <typename T> int is() const { // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename ::llvm::PointerUnionTypeSelector< + using Ty = typename ::llvm::PointerUnionTypeSelector< PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return - Ty; + ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return; return Ty(Val).template is<T>(); } @@ -281,10 +284,9 @@ public: template <typename T> T get() const { assert(is<T>() && "Invalid accessor called"); // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename ::llvm::PointerUnionTypeSelector< + using Ty = typename ::llvm::PointerUnionTypeSelector< PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return - Ty; + ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return; return Ty(Val).template get<T>(); } @@ -328,8 +330,7 @@ public: // Teach SmallPtrSet that PointerUnion3 is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. template <typename PT1, typename PT2, typename PT3> -class PointerLikeTypeTraits<PointerUnion3<PT1, PT2, PT3>> { -public: +struct PointerLikeTypeTraits<PointerUnion3<PT1, PT2, PT3>> { static inline void *getAsVoidPointer(const PointerUnion3<PT1, PT2, PT3> &P) { return P.getOpaqueValue(); } @@ -350,16 +351,15 @@ public: template <typename PT1, typename PT2, typename PT3, typename PT4> class PointerUnion4 { public: - typedef PointerUnion<PT1, PT2> InnerUnion1; - typedef PointerUnion<PT3, PT4> InnerUnion2; - typedef PointerUnion<InnerUnion1, InnerUnion2> ValTy; + using InnerUnion1 = PointerUnion<PT1, PT2>; + using InnerUnion2 = PointerUnion<PT3, PT4>; + using ValTy = PointerUnion<InnerUnion1, InnerUnion2>; private: ValTy Val; public: PointerUnion4() = default; - PointerUnion4(PT1 V) { Val = InnerUnion1(V); } PointerUnion4(PT2 V) { Val = InnerUnion1(V); } PointerUnion4(PT3 V) { Val = InnerUnion2(V); } @@ -373,9 +373,10 @@ public: /// Test if the Union currently holds the type matching T. template <typename T> int is() const { // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< - PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, + ::llvm::PointerUnionTypeSelector<PT2, T, InnerUnion1, + InnerUnion2>>::Return; return Val.template is<Ty>() && Val.template get<Ty>().template is<T>(); } @@ -385,9 +386,10 @@ public: template <typename T> T get() const { assert(is<T>() && "Invalid accessor called"); // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< - PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + using Ty = typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, + ::llvm::PointerUnionTypeSelector<PT2, T, InnerUnion1, + InnerUnion2>>::Return; return Val.template get<Ty>().template get<T>(); } @@ -435,8 +437,7 @@ public: // Teach SmallPtrSet that PointerUnion4 is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. template <typename PT1, typename PT2, typename PT3, typename PT4> -class PointerLikeTypeTraits<PointerUnion4<PT1, PT2, PT3, PT4>> { -public: +struct PointerLikeTypeTraits<PointerUnion4<PT1, PT2, PT3, PT4>> { static inline void * getAsVoidPointer(const PointerUnion4<PT1, PT2, PT3, PT4> &P) { return P.getOpaqueValue(); @@ -455,18 +456,21 @@ public: // Teach DenseMap how to use PointerUnions as keys. template <typename T, typename U> struct DenseMapInfo<PointerUnion<T, U>> { - typedef PointerUnion<T, U> Pair; - typedef DenseMapInfo<T> FirstInfo; - typedef DenseMapInfo<U> SecondInfo; + using Pair = PointerUnion<T, U>; + using FirstInfo = DenseMapInfo<T>; + using SecondInfo = DenseMapInfo<U>; static inline Pair getEmptyKey() { return Pair(FirstInfo::getEmptyKey()); } + static inline Pair getTombstoneKey() { return Pair(FirstInfo::getTombstoneKey()); } + static unsigned getHashValue(const Pair &PairVal) { intptr_t key = (intptr_t)PairVal.getOpaqueValue(); return DenseMapInfo<intptr_t>::getHashValue(key); } + static bool isEqual(const Pair &LHS, const Pair &RHS) { return LHS.template is<T>() == RHS.template is<T>() && (LHS.template is<T>() ? FirstInfo::isEqual(LHS.template get<T>(), diff --git a/contrib/llvm/include/llvm/ADT/STLExtras.h b/contrib/llvm/include/llvm/ADT/STLExtras.h index 83f289c42a23..bcd992b4a716 100644 --- a/contrib/llvm/include/llvm/ADT/STLExtras.h +++ b/contrib/llvm/include/llvm/ADT/STLExtras.h @@ -17,23 +17,24 @@ #ifndef LLVM_ADT_STLEXTRAS_H #define LLVM_ADT_STLEXTRAS_H -#include <algorithm> // for std::all_of +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/ErrorHandling.h" +#include <algorithm> #include <cassert> -#include <cstddef> // for std::size_t -#include <cstdlib> // for qsort +#include <cstddef> +#include <cstdint> +#include <cstdlib> #include <functional> +#include <initializer_list> #include <iterator> #include <limits> #include <memory> #include <tuple> -#include <utility> // for std::pair - -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/iterator.h" -#include "llvm/ADT/iterator_range.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" +#include <type_traits> +#include <utility> namespace llvm { @@ -50,14 +51,15 @@ template <typename RangeT> using ValueOfRange = typename std::remove_reference<decltype( *std::begin(std::declval<RangeT &>()))>::type; -} // End detail namespace +} // end namespace detail //===----------------------------------------------------------------------===// // Extra additions to <functional> //===----------------------------------------------------------------------===// -template<class Ty> -struct identity : public std::unary_function<Ty, Ty> { +template <class Ty> struct identity { + using argument_type = Ty; + Ty &operator()(Ty &self) const { return self; } @@ -66,15 +68,13 @@ struct identity : public std::unary_function<Ty, Ty> { } }; -template<class Ty> -struct less_ptr : public std::binary_function<Ty, Ty, bool> { +template <class Ty> struct less_ptr { bool operator()(const Ty* left, const Ty* right) const { return *left < *right; } }; -template<class Ty> -struct greater_ptr : public std::binary_function<Ty, Ty, bool> { +template <class Ty> struct greater_ptr { bool operator()(const Ty* left, const Ty* right) const { return *right < *left; } @@ -90,7 +90,7 @@ template<typename Fn> class function_ref; template<typename Ret, typename ...Params> class function_ref<Ret(Params...)> { - Ret (*callback)(intptr_t callable, Params ...params); + Ret (*callback)(intptr_t callable, Params ...params) = nullptr; intptr_t callable; template<typename Callable> @@ -100,7 +100,7 @@ class function_ref<Ret(Params...)> { } public: - function_ref() : callback(nullptr) {} + function_ref() = default; template <typename Callable> function_ref(Callable &&callable, @@ -109,6 +109,7 @@ public: function_ref>::value>::type * = nullptr) : callback(callback_fn<typename std::remove_reference<Callable>::type>), callable(reinterpret_cast<intptr_t>(&callable)) {} + Ret operator()(Params ...params) const { return callback(callable, std::forward<Params>(params)...); } @@ -120,114 +121,95 @@ public: // delete on something. It is used like this: // // for_each(V.begin(), B.end(), deleter<Interval>); -// template <class T> inline void deleter(T *Ptr) { delete Ptr; } - - //===----------------------------------------------------------------------===// // Extra additions to <iterator> //===----------------------------------------------------------------------===// -// mapped_iterator - This is a simple iterator adapter that causes a function to -// be applied whenever operator* is invoked on the iterator. -// -template <class RootIt, class UnaryFunc> -class mapped_iterator { - RootIt current; - UnaryFunc Fn; -public: - typedef typename std::iterator_traits<RootIt>::iterator_category - iterator_category; - typedef typename std::iterator_traits<RootIt>::difference_type - difference_type; - typedef decltype(std::declval<UnaryFunc>()(*std::declval<RootIt>())) - value_type; +namespace adl_detail { + +using std::begin; - typedef void pointer; - //typedef typename UnaryFunc::result_type *pointer; - typedef void reference; // Can't modify value returned by fn +template <typename ContainerTy> +auto adl_begin(ContainerTy &&container) + -> decltype(begin(std::forward<ContainerTy>(container))) { + return begin(std::forward<ContainerTy>(container)); +} - typedef RootIt iterator_type; +using std::end; - inline const RootIt &getCurrent() const { return current; } - inline const UnaryFunc &getFunc() const { return Fn; } +template <typename ContainerTy> +auto adl_end(ContainerTy &&container) + -> decltype(end(std::forward<ContainerTy>(container))) { + return end(std::forward<ContainerTy>(container)); +} - inline explicit mapped_iterator(const RootIt &I, UnaryFunc F) - : current(I), Fn(F) {} +using std::swap; - inline value_type operator*() const { // All this work to do this - return Fn(*current); // little change - } +template <typename T> +void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval<T>(), + std::declval<T>()))) { + swap(std::forward<T>(lhs), std::forward<T>(rhs)); +} - mapped_iterator &operator++() { - ++current; - return *this; - } - mapped_iterator &operator--() { - --current; - return *this; - } - mapped_iterator operator++(int) { - mapped_iterator __tmp = *this; - ++current; - return __tmp; - } - mapped_iterator operator--(int) { - mapped_iterator __tmp = *this; - --current; - return __tmp; - } - mapped_iterator operator+(difference_type n) const { - return mapped_iterator(current + n, Fn); - } - mapped_iterator &operator+=(difference_type n) { - current += n; - return *this; - } - mapped_iterator operator-(difference_type n) const { - return mapped_iterator(current - n, Fn); - } - mapped_iterator &operator-=(difference_type n) { - current -= n; - return *this; - } - reference operator[](difference_type n) const { return *(*this + n); } +} // end namespace adl_detail - bool operator!=(const mapped_iterator &X) const { return !operator==(X); } - bool operator==(const mapped_iterator &X) const { - return current == X.current; - } - bool operator<(const mapped_iterator &X) const { return current < X.current; } +template <typename ContainerTy> +auto adl_begin(ContainerTy &&container) + -> decltype(adl_detail::adl_begin(std::forward<ContainerTy>(container))) { + return adl_detail::adl_begin(std::forward<ContainerTy>(container)); +} - difference_type operator-(const mapped_iterator &X) const { - return current - X.current; - } -}; +template <typename ContainerTy> +auto adl_end(ContainerTy &&container) + -> decltype(adl_detail::adl_end(std::forward<ContainerTy>(container))) { + return adl_detail::adl_end(std::forward<ContainerTy>(container)); +} -template <class Iterator, class Func> -inline mapped_iterator<Iterator, Func> -operator+(typename mapped_iterator<Iterator, Func>::difference_type N, - const mapped_iterator<Iterator, Func> &X) { - return mapped_iterator<Iterator, Func>(X.getCurrent() - N, X.getFunc()); +template <typename T> +void adl_swap(T &&lhs, T &&rhs) noexcept( + noexcept(adl_detail::adl_swap(std::declval<T>(), std::declval<T>()))) { + adl_detail::adl_swap(std::forward<T>(lhs), std::forward<T>(rhs)); } +// mapped_iterator - This is a simple iterator adapter that causes a function to +// be applied whenever operator* is invoked on the iterator. + +template <typename ItTy, typename FuncTy, + typename FuncReturnTy = + decltype(std::declval<FuncTy>()(*std::declval<ItTy>()))> +class mapped_iterator + : public iterator_adaptor_base< + mapped_iterator<ItTy, FuncTy>, ItTy, + typename std::iterator_traits<ItTy>::iterator_category, + typename std::remove_reference<FuncReturnTy>::type> { +public: + mapped_iterator(ItTy U, FuncTy F) + : mapped_iterator::iterator_adaptor_base(std::move(U)), F(std::move(F)) {} + + ItTy getCurrent() { return this->I; } + + FuncReturnTy operator*() { return F(*this->I); } + +private: + FuncTy F; +}; // map_iterator - Provide a convenient way to create mapped_iterators, just like // make_pair is useful for creating pairs... -// template <class ItTy, class FuncTy> -inline mapped_iterator<ItTy, FuncTy> map_iterator(const ItTy &I, FuncTy F) { - return mapped_iterator<ItTy, FuncTy>(I, F); +inline mapped_iterator<ItTy, FuncTy> map_iterator(ItTy I, FuncTy F) { + return mapped_iterator<ItTy, FuncTy>(std::move(I), std::move(F)); } /// Helper to determine if type T has a member called rbegin(). template <typename Ty> class has_rbegin_impl { - typedef char yes[1]; - typedef char no[2]; + using yes = char[1]; + using no = char[2]; template <typename Inner> static yes& test(Inner *I, decltype(I->rbegin()) * = nullptr); @@ -365,12 +347,13 @@ template <size_t... I> struct index_sequence; template <class... Ts> struct index_sequence_for; namespace detail { + using std::declval; // We have to alias this since inlining the actual type at the usage site // in the parameter list of iterator_facade_base<> below ICEs MSVC 2017. template<typename... Iters> struct ZipTupleType { - typedef std::tuple<decltype(*declval<Iters>())...> type; + using type = std::tuple<decltype(*declval<Iters>())...>; }; template <typename ZipType, typename... Iters> @@ -456,11 +439,11 @@ class zip_shortest : public zip_common<zip_shortest<Iters...>, Iters...> { public: using Base = zip_common<zip_shortest<Iters...>, Iters...>; + zip_shortest(Iters &&... ts) : Base(std::forward<Iters>(ts)...) {} + bool operator==(const zip_shortest<Iters...> &other) const { return !test(other, index_sequence_for<Iters...>{}); } - - zip_shortest(Iters &&... ts) : Base(std::forward<Iters>(ts)...) {} }; template <template <typename...> class ItType, typename... Args> class zippy { @@ -483,11 +466,13 @@ private: } public: + zippy(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} + iterator begin() const { return begin_impl(index_sequence_for<Args...>{}); } iterator end() const { return end_impl(index_sequence_for<Args...>{}); } - zippy(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} }; -} // End detail namespace + +} // end namespace detail /// zip iterator for two or more iteratable types. template <typename T, typename U, typename... Args> @@ -520,7 +505,7 @@ template <typename ValueT, typename... IterTs> class concat_iterator : public iterator_facade_base<concat_iterator<ValueT, IterTs...>, std::forward_iterator_tag, ValueT> { - typedef typename concat_iterator::iterator_facade_base BaseT; + using BaseT = typename concat_iterator::iterator_facade_base; /// We store both the current and end iterators for each concatenated /// sequence in a tuple of pairs. @@ -597,6 +582,7 @@ public: : IterPairs({std::begin(Ranges), std::end(Ranges)}...) {} using BaseT::operator++; + concat_iterator &operator++() { increment(index_sequence_for<IterTs...>()); return *this; @@ -610,6 +596,7 @@ public: }; namespace detail { + /// Helper to store a sequence of ranges being concatenated and access them. /// /// This is designed to facilitate providing actual storage when temporaries @@ -617,9 +604,9 @@ namespace detail { /// based for loops. template <typename ValueT, typename... RangeTs> class concat_range { public: - typedef concat_iterator<ValueT, - decltype(std::begin(std::declval<RangeTs &>()))...> - iterator; + using iterator = + concat_iterator<ValueT, + decltype(std::begin(std::declval<RangeTs &>()))...>; private: std::tuple<RangeTs...> Ranges; @@ -633,12 +620,14 @@ private: } public: - iterator begin() { return begin_impl(index_sequence_for<RangeTs...>{}); } - iterator end() { return end_impl(index_sequence_for<RangeTs...>{}); } concat_range(RangeTs &&... Ranges) : Ranges(std::forward<RangeTs>(Ranges)...) {} + + iterator begin() { return begin_impl(index_sequence_for<RangeTs...>{}); } + iterator end() { return end_impl(index_sequence_for<RangeTs...>{}); } }; -} + +} // end namespace detail /// Concatenated range across two or more ranges. /// @@ -675,7 +664,7 @@ struct less_second { /// \brief Represents a compile-time sequence of integers. template <class T, T... I> struct integer_sequence { - typedef T value_type; + using value_type = T; static constexpr size_t size() { return sizeof...(I); } }; @@ -752,7 +741,6 @@ inline int (*get_array_pod_sort_comparator(const T &)) return array_pod_sort_comparator<T>; } - /// array_pod_sort - This sorts an array with the specified start and end /// extent. This is just like std::sort, except that it calls qsort instead of /// using an inlined template. qsort is slightly slower than std::sort, but @@ -812,96 +800,109 @@ void DeleteContainerSeconds(Container &C) { C.clear(); } +/// Provide wrappers to std::for_each which take ranges instead of having to +/// pass begin/end explicitly. +template <typename R, typename UnaryPredicate> +UnaryPredicate for_each(R &&Range, UnaryPredicate P) { + return std::for_each(adl_begin(Range), adl_end(Range), P); +} + /// Provide wrappers to std::all_of which take ranges instead of having to pass /// begin/end explicitly. template <typename R, typename UnaryPredicate> bool all_of(R &&Range, UnaryPredicate P) { - return std::all_of(std::begin(Range), std::end(Range), P); + return std::all_of(adl_begin(Range), adl_end(Range), P); } /// Provide wrappers to std::any_of which take ranges instead of having to pass /// begin/end explicitly. template <typename R, typename UnaryPredicate> bool any_of(R &&Range, UnaryPredicate P) { - return std::any_of(std::begin(Range), std::end(Range), P); + return std::any_of(adl_begin(Range), adl_end(Range), P); } /// Provide wrappers to std::none_of which take ranges instead of having to pass /// begin/end explicitly. template <typename R, typename UnaryPredicate> bool none_of(R &&Range, UnaryPredicate P) { - return std::none_of(std::begin(Range), std::end(Range), P); + return std::none_of(adl_begin(Range), adl_end(Range), P); } /// Provide wrappers to std::find which take ranges instead of having to pass /// begin/end explicitly. template <typename R, typename T> -auto find(R &&Range, const T &Val) -> decltype(std::begin(Range)) { - return std::find(std::begin(Range), std::end(Range), Val); +auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range)) { + return std::find(adl_begin(Range), adl_end(Range), Val); } /// Provide wrappers to std::find_if which take ranges instead of having to pass /// begin/end explicitly. template <typename R, typename UnaryPredicate> -auto find_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range)) { - return std::find_if(std::begin(Range), std::end(Range), P); +auto find_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) { + return std::find_if(adl_begin(Range), adl_end(Range), P); } template <typename R, typename UnaryPredicate> -auto find_if_not(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range)) { - return std::find_if_not(std::begin(Range), std::end(Range), P); +auto find_if_not(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) { + return std::find_if_not(adl_begin(Range), adl_end(Range), P); } /// Provide wrappers to std::remove_if which take ranges instead of having to /// pass begin/end explicitly. template <typename R, typename UnaryPredicate> -auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range)) { - return std::remove_if(std::begin(Range), std::end(Range), P); +auto remove_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) { + return std::remove_if(adl_begin(Range), adl_end(Range), P); } /// Provide wrappers to std::copy_if which take ranges instead of having to /// pass begin/end explicitly. template <typename R, typename OutputIt, typename UnaryPredicate> OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P) { - return std::copy_if(std::begin(Range), std::end(Range), Out, P); + return std::copy_if(adl_begin(Range), adl_end(Range), Out, P); } /// Wrapper function around std::find to detect if an element exists /// in a container. template <typename R, typename E> bool is_contained(R &&Range, const E &Element) { - return std::find(std::begin(Range), std::end(Range), Element) != - std::end(Range); + return std::find(adl_begin(Range), adl_end(Range), Element) != adl_end(Range); } /// Wrapper function around std::count to count the number of times an element /// \p Element occurs in the given range \p Range. template <typename R, typename E> -auto count(R &&Range, const E &Element) -> typename std::iterator_traits< - decltype(std::begin(Range))>::difference_type { - return std::count(std::begin(Range), std::end(Range), Element); +auto count(R &&Range, const E &Element) -> + typename std::iterator_traits<decltype(adl_begin(Range))>::difference_type { + return std::count(adl_begin(Range), adl_end(Range), Element); } /// Wrapper function around std::count_if to count the number of times an /// element satisfying a given predicate occurs in a range. template <typename R, typename UnaryPredicate> -auto count_if(R &&Range, UnaryPredicate P) -> typename std::iterator_traits< - decltype(std::begin(Range))>::difference_type { - return std::count_if(std::begin(Range), std::end(Range), P); +auto count_if(R &&Range, UnaryPredicate P) -> + typename std::iterator_traits<decltype(adl_begin(Range))>::difference_type { + return std::count_if(adl_begin(Range), adl_end(Range), P); } /// Wrapper function around std::transform to apply a function to a range and /// store the result elsewhere. template <typename R, typename OutputIt, typename UnaryPredicate> OutputIt transform(R &&Range, OutputIt d_first, UnaryPredicate P) { - return std::transform(std::begin(Range), std::end(Range), d_first, P); + return std::transform(adl_begin(Range), adl_end(Range), d_first, P); } /// Provide wrappers to std::partition which take ranges instead of having to /// pass begin/end explicitly. template <typename R, typename UnaryPredicate> -auto partition(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range)) { - return std::partition(std::begin(Range), std::end(Range), P); +auto partition(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) { + return std::partition(adl_begin(Range), adl_end(Range), P); +} + +/// Provide wrappers to std::lower_bound which take ranges instead of having to +/// pass begin/end explicitly. +template <typename R, typename ForwardIt> +auto lower_bound(R &&Range, ForwardIt I) -> decltype(adl_begin(Range)) { + return std::lower_bound(adl_begin(Range), adl_end(Range), I); } /// \brief Given a range of type R, iterate the entire range and return a @@ -910,7 +911,7 @@ auto partition(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range)) { template <unsigned Size, typename R> SmallVector<typename std::remove_const<detail::ValueOfRange<R>>::type, Size> to_vector(R &&Range) { - return {std::begin(Range), std::end(Range)}; + return {adl_begin(Range), adl_end(Range)}; } /// Provide a container algorithm similar to C++ Library Fundamentals v2's @@ -995,6 +996,7 @@ struct equal { /// operands. template <typename T> struct deref { T func; + // Could be further improved to cope with non-derivable functors and // non-binary functors (should be a variadic template member function // operator()). @@ -1007,12 +1009,13 @@ template <typename T> struct deref { }; namespace detail { + template <typename R> class enumerator_iter; template <typename R> struct result_pair { friend class enumerator_iter<R>; - result_pair() : Index(-1) {} + result_pair() = default; result_pair(std::size_t Index, IterOfRange<R> Iter) : Index(Index), Iter(Iter) {} @@ -1027,7 +1030,7 @@ template <typename R> struct result_pair { ValueOfRange<R> &value() { return *Iter; } private: - std::size_t Index; + std::size_t Index = std::numeric_limits<std::size_t>::max(); IterOfRange<R> Iter; }; @@ -1042,7 +1045,7 @@ class enumerator_iter public: explicit enumerator_iter(IterOfRange<R> EndIter) - : Result(std::numeric_limits<size_t>::max(), EndIter) { } + : Result(std::numeric_limits<size_t>::max(), EndIter) {} enumerator_iter(std::size_t Index, IterOfRange<R> Iter) : Result(Index, Iter) {} @@ -1080,6 +1083,7 @@ public: enumerator_iter<R> begin() { return enumerator_iter<R>(0, std::begin(TheRange)); } + enumerator_iter<R> end() { return enumerator_iter<R>(std::end(TheRange)); } @@ -1087,7 +1091,8 @@ public: private: R TheRange; }; -} + +} // end namespace detail /// Given an input range, returns a new range whose values are are pair (A,B) /// such that A is the 0-based index of the item in the sequence, and B is @@ -1109,12 +1114,14 @@ template <typename R> detail::enumerator<R> enumerate(R &&TheRange) { } namespace detail { + template <typename F, typename Tuple, std::size_t... I> auto apply_tuple_impl(F &&f, Tuple &&t, index_sequence<I...>) -> decltype(std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...)) { return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...); } -} + +} // end namespace detail /// Given an input tuple (a1, a2, ..., an), pass the arguments of the /// tuple variadically to f as if by calling f(a1, a2, ..., an) and @@ -1130,6 +1137,7 @@ auto apply_tuple(F &&f, Tuple &&t) -> decltype(detail::apply_tuple_impl( return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t), Indices{}); } -} // End llvm namespace -#endif +} // end namespace llvm + +#endif // LLVM_ADT_STLEXTRAS_H diff --git a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h index 4e8a2490ee3c..78ea613af693 100644 --- a/contrib/llvm/include/llvm/ADT/SmallPtrSet.h +++ b/contrib/llvm/include/llvm/ADT/SmallPtrSet.h @@ -15,8 +15,8 @@ #ifndef LLVM_ADT_SMALLPTRSET_H #define LLVM_ADT_SMALLPTRSET_H +#include "llvm/ADT/EpochTracker.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/PointerLikeTypeTraits.h" #include "llvm/Support/ReverseIteration.h" #include "llvm/Support/type_traits.h" #include <cassert> @@ -47,7 +47,7 @@ namespace llvm { /// (-2), to allow deletion. The hash table is resized when the table is 3/4 or /// more. When this happens, the table is doubled in size. /// -class SmallPtrSetImplBase { +class SmallPtrSetImplBase : public DebugEpochBase { friend class SmallPtrSetIteratorImpl; protected: @@ -93,6 +93,7 @@ public: size_type size() const { return NumNonEmpty - NumTombstones; } void clear() { + incrementEpoch(); // If the capacity of the array is huge, and the # elements used is small, // shrink the array. if (!isSmall()) { @@ -139,12 +140,14 @@ protected: if (LastTombstone != nullptr) { *LastTombstone = Ptr; --NumTombstones; + incrementEpoch(); return std::make_pair(LastTombstone, true); } // Nope, there isn't. If we stay small, just 'pushback' now. if (NumNonEmpty < CurArraySize) { SmallArray[NumNonEmpty++] = Ptr; + incrementEpoch(); return std::make_pair(SmallArray + (NumNonEmpty - 1), true); } // Otherwise, hit the big set case, which will call grow. @@ -224,12 +227,10 @@ protected: public: explicit SmallPtrSetIteratorImpl(const void *const *BP, const void*const *E) : Bucket(BP), End(E) { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (ReverseIterate<bool>::value) { + if (shouldReverseIterate()) { RetreatIfNotValid(); return; } -#endif AdvanceIfNotValid(); } @@ -251,7 +252,6 @@ protected: *Bucket == SmallPtrSetImplBase::getTombstoneMarker())) ++Bucket; } -#if LLVM_ENABLE_ABI_BREAKING_CHECKS void RetreatIfNotValid() { assert(Bucket >= End); while (Bucket != End && @@ -260,12 +260,12 @@ protected: --Bucket; } } -#endif }; /// SmallPtrSetIterator - This implements a const_iterator for SmallPtrSet. -template<typename PtrTy> -class SmallPtrSetIterator : public SmallPtrSetIteratorImpl { +template <typename PtrTy> +class SmallPtrSetIterator : public SmallPtrSetIteratorImpl, + DebugEpochBase::HandleBase { using PtrTraits = PointerLikeTypeTraits<PtrTy>; public: @@ -275,30 +275,29 @@ public: using difference_type = std::ptrdiff_t; using iterator_category = std::forward_iterator_tag; - explicit SmallPtrSetIterator(const void *const *BP, const void *const *E) - : SmallPtrSetIteratorImpl(BP, E) {} + explicit SmallPtrSetIterator(const void *const *BP, const void *const *E, + const DebugEpochBase &Epoch) + : SmallPtrSetIteratorImpl(BP, E), DebugEpochBase::HandleBase(&Epoch) {} // Most methods provided by baseclass. const PtrTy operator*() const { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (ReverseIterate<bool>::value) { + assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate()) { assert(Bucket > End); return PtrTraits::getFromVoidPointer(const_cast<void *>(Bucket[-1])); } -#endif assert(Bucket < End); return PtrTraits::getFromVoidPointer(const_cast<void*>(*Bucket)); } inline SmallPtrSetIterator& operator++() { // Preincrement -#if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (ReverseIterate<bool>::value) { + assert(isHandleInSync() && "invalid iterator access!"); + if (shouldReverseIterate()) { --Bucket; RetreatIfNotValid(); return *this; } -#endif ++Bucket; AdvanceIfNotValid(); return *this; @@ -396,10 +395,8 @@ public: } iterator begin() const { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (ReverseIterate<bool>::value) + if (shouldReverseIterate()) return makeIterator(EndPointer() - 1); -#endif return makeIterator(CurArray); } iterator end() const { return makeIterator(EndPointer()); } @@ -407,11 +404,9 @@ public: private: /// Create an iterator that dereferences to same place as the given pointer. iterator makeIterator(const void *const *P) const { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (ReverseIterate<bool>::value) - return iterator(P == EndPointer() ? CurArray : P + 1, CurArray); -#endif - return iterator(P, EndPointer()); + if (shouldReverseIterate()) + return iterator(P == EndPointer() ? CurArray : P + 1, CurArray, *this); + return iterator(P, EndPointer(), *this); } }; diff --git a/contrib/llvm/include/llvm/ADT/SmallVector.h b/contrib/llvm/include/llvm/ADT/SmallVector.h index bf2a62f43aff..a9ac98d1ad4c 100644 --- a/contrib/llvm/include/llvm/ADT/SmallVector.h +++ b/contrib/llvm/include/llvm/ADT/SmallVector.h @@ -19,6 +19,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/type_traits.h" +#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -238,6 +239,8 @@ void SmallVectorTemplateBase<T, isPodLike>::grow(size_t MinSize) { if (NewCapacity < MinSize) NewCapacity = MinSize; T *NewElts = static_cast<T*>(malloc(NewCapacity*sizeof(T))); + if (NewElts == nullptr) + report_bad_alloc_error("Allocation of SmallVector element failed."); // Move the elements over. this->uninitialized_move(this->begin(), this->end(), NewElts); @@ -924,8 +927,8 @@ public: } }; -template<typename T, unsigned N> -static inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { +template <typename T, unsigned N> +inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { return X.capacity_in_bytes(); } diff --git a/contrib/llvm/include/llvm/ADT/StringExtras.h b/contrib/llvm/include/llvm/ADT/StringExtras.h index cc32bf43f29c..60652f8c55c5 100644 --- a/contrib/llvm/include/llvm/ADT/StringExtras.h +++ b/contrib/llvm/include/llvm/ADT/StringExtras.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" #include <cassert> #include <cstddef> #include <cstdint> @@ -33,33 +34,65 @@ class raw_ostream; /// hexdigit - Return the hexadecimal character for the /// given number \p X (which should be less than 16). -static inline char hexdigit(unsigned X, bool LowerCase = false) { +inline char hexdigit(unsigned X, bool LowerCase = false) { const char HexChar = LowerCase ? 'a' : 'A'; return X < 10 ? '0' + X : HexChar + X - 10; } /// Construct a string ref from a boolean. -static inline StringRef toStringRef(bool B) { - return StringRef(B ? "true" : "false"); -} +inline StringRef toStringRef(bool B) { return StringRef(B ? "true" : "false"); } /// Construct a string ref from an array ref of unsigned chars. -static inline StringRef toStringRef(ArrayRef<uint8_t> Input) { +inline StringRef toStringRef(ArrayRef<uint8_t> Input) { return StringRef(reinterpret_cast<const char *>(Input.begin()), Input.size()); } +/// Construct a string ref from an array ref of unsigned chars. +inline ArrayRef<uint8_t> arrayRefFromStringRef(StringRef Input) { + return {Input.bytes_begin(), Input.bytes_end()}; +} + /// Interpret the given character \p C as a hexadecimal digit and return its /// value. /// /// If \p C is not a valid hex digit, -1U is returned. -static inline unsigned hexDigitValue(char C) { +inline unsigned hexDigitValue(char C) { if (C >= '0' && C <= '9') return C-'0'; if (C >= 'a' && C <= 'f') return C-'a'+10U; if (C >= 'A' && C <= 'F') return C-'A'+10U; return -1U; } -static inline std::string utohexstr(uint64_t X, bool LowerCase = false) { +/// Checks if character \p C is one of the 10 decimal digits. +inline bool isDigit(char C) { return C >= '0' && C <= '9'; } + +/// Checks if character \p C is a hexadecimal numeric character. +inline bool isHexDigit(char C) { return hexDigitValue(C) != -1U; } + +/// Checks if character \p C is a valid letter as classified by "C" locale. +inline bool isAlpha(char C) { + return ('a' <= C && C <= 'z') || ('A' <= C && C <= 'Z'); +} + +/// Checks whether character \p C is either a decimal digit or an uppercase or +/// lowercase letter as classified by "C" locale. +inline bool isAlnum(char C) { return isAlpha(C) || isDigit(C); } + +/// Returns the corresponding lowercase character if \p x is uppercase. +inline char toLower(char x) { + if (x >= 'A' && x <= 'Z') + return x - 'A' + 'a'; + return x; +} + +/// Returns the corresponding uppercase character if \p x is lowercase. +inline char toUpper(char x) { + if (x >= 'a' && x <= 'z') + return x - 'a' + 'A'; + return x; +} + +inline std::string utohexstr(uint64_t X, bool LowerCase = false) { char Buffer[17]; char *BufPtr = std::end(Buffer); @@ -94,7 +127,7 @@ inline std::string toHex(ArrayRef<uint8_t> Input) { return toHex(toStringRef(Input)); } -static inline uint8_t hexFromNibbles(char MSB, char LSB) { +inline uint8_t hexFromNibbles(char MSB, char LSB) { unsigned U1 = hexDigitValue(MSB); unsigned U2 = hexDigitValue(LSB); assert(U1 != -1U && U2 != -1U); @@ -104,7 +137,7 @@ static inline uint8_t hexFromNibbles(char MSB, char LSB) { /// Convert hexadecimal string \p Input to its binary representation. /// The return string is half the size of \p Input. -static inline std::string fromHex(StringRef Input) { +inline std::string fromHex(StringRef Input) { if (Input.empty()) return std::string(); @@ -157,7 +190,7 @@ inline bool to_float(const Twine &T, long double &Num) { return detail::to_float(T, Num, strtold); } -static inline std::string utostr(uint64_t X, bool isNeg = false) { +inline std::string utostr(uint64_t X, bool isNeg = false) { char Buffer[21]; char *BufPtr = std::end(Buffer); @@ -172,7 +205,7 @@ static inline std::string utostr(uint64_t X, bool isNeg = false) { return std::string(BufPtr, std::end(Buffer)); } -static inline std::string itostr(int64_t X) { +inline std::string itostr(int64_t X) { if (X < 0) return utostr(static_cast<uint64_t>(-X), true); else @@ -206,14 +239,14 @@ void SplitString(StringRef Source, // FIXME: Investigate whether a modified bernstein hash function performs // better: http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx // X*33+c -> X*33^c -static inline unsigned HashString(StringRef Str, unsigned Result = 0) { +inline unsigned HashString(StringRef Str, unsigned Result = 0) { for (StringRef::size_type i = 0, e = Str.size(); i != e; ++i) Result = Result * 33 + (unsigned char)Str[i]; return Result; } /// Returns the English suffix for an ordinal integer (-st, -nd, -rd, -th). -static inline StringRef getOrdinalSuffix(unsigned Val) { +inline StringRef getOrdinalSuffix(unsigned Val) { // It is critically important that we do this perfectly for // user-written sequences with over 100 elements. switch (Val % 100) { @@ -235,6 +268,9 @@ static inline StringRef getOrdinalSuffix(unsigned Val) { /// it if it is not printable or if it is an escape char. void PrintEscapedString(StringRef Name, raw_ostream &Out); +/// printLowerCase - Print each character as lowercase if it is uppercase. +void printLowerCase(StringRef String, raw_ostream &Out); + namespace detail { template <typename IteratorT> diff --git a/contrib/llvm/include/llvm/ADT/StringMap.h b/contrib/llvm/include/llvm/ADT/StringMap.h index d573148665a1..6c2830b44914 100644 --- a/contrib/llvm/include/llvm/ADT/StringMap.h +++ b/contrib/llvm/include/llvm/ADT/StringMap.h @@ -19,6 +19,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <cassert> #include <cstdint> @@ -165,6 +166,9 @@ public: StringMapEntry *NewItem = static_cast<StringMapEntry*>(Allocator.Allocate(AllocSize,Alignment)); + if (NewItem == nullptr) + report_bad_alloc_error("Allocation of StringMap entry failed."); + // Construct the value. new (NewItem) StringMapEntry(KeyLength, std::forward<InitTy>(InitVals)...); diff --git a/contrib/llvm/include/llvm/ADT/TinyPtrVector.h b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h index 79740713f75b..73573d65e2b3 100644 --- a/contrib/llvm/include/llvm/ADT/TinyPtrVector.h +++ b/contrib/llvm/include/llvm/ADT/TinyPtrVector.h @@ -97,6 +97,7 @@ public: if (RHS.Val.template is<EltTy>()) { V->clear(); V->push_back(RHS.front()); + RHS.Val = (EltTy)nullptr; return *this; } delete V; diff --git a/contrib/llvm/include/llvm/ADT/Triple.h b/contrib/llvm/include/llvm/ADT/Triple.h index cd560658ca4e..74fc8eb8ccbf 100644 --- a/contrib/llvm/include/llvm/ADT/Triple.h +++ b/contrib/llvm/include/llvm/ADT/Triple.h @@ -50,6 +50,7 @@ public: armeb, // ARM (big endian): armeb aarch64, // AArch64 (little endian): aarch64 aarch64_be, // AArch64 (big endian): aarch64_be + arc, // ARC: Synopsys ARC avr, // AVR: Atmel AVR microcontroller bpfel, // eBPF or extended BPF or 64-bit BPF (little endian) bpfeb, // eBPF or extended BPF or 64-bit BPF (big endian) @@ -100,6 +101,7 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v8_3a, ARMSubArch_v8_2a, ARMSubArch_v8_1a, ARMSubArch_v8, @@ -167,7 +169,6 @@ public: RTEMS, NaCl, // Native Client CNK, // BG/P Compute-Node Kernel - Bitrig, AIX, CUDA, // NVIDIA CUDA NVCL, // NVIDIA OpenCL @@ -178,12 +179,14 @@ public: WatchOS, // Apple watchOS Mesa3D, Contiki, - LastOSType = Contiki + AMDPAL, // AMD PAL Runtime + LastOSType = AMDPAL }; enum EnvironmentType { UnknownEnvironment, GNU, + GNUABIN32, GNUABI64, GNUEABI, GNUEABIHF, @@ -202,7 +205,8 @@ public: AMDOpenCL, CoreCLR, OpenCL, - LastEnvironmentType = OpenCL + Simulator, // Simulator variants of other systems, e.g., Apple's iOS + LastEnvironmentType = Simulator }; enum ObjectFormatType { UnknownObjectFormat, @@ -467,6 +471,10 @@ public: return isMacOSX() || isiOS() || isWatchOS(); } + bool isSimulatorEnvironment() const { + return getEnvironment() == Triple::Simulator; + } + bool isOSNetBSD() const { return getOS() == Triple::NetBSD; } @@ -489,25 +497,28 @@ public: return getOS() == Triple::Solaris; } - bool isOSBitrig() const { - return getOS() == Triple::Bitrig; - } - bool isOSIAMCU() const { return getOS() == Triple::ELFIAMCU; } + bool isOSUnknown() const { return getOS() == Triple::UnknownOS; } + bool isGNUEnvironment() const { EnvironmentType Env = getEnvironment(); - return Env == Triple::GNU || Env == Triple::GNUABI64 || - Env == Triple::GNUEABI || Env == Triple::GNUEABIHF || - Env == Triple::GNUX32; + return Env == Triple::GNU || Env == Triple::GNUABIN32 || + Env == Triple::GNUABI64 || Env == Triple::GNUEABI || + Env == Triple::GNUEABIHF || Env == Triple::GNUX32; } bool isOSContiki() const { return getOS() == Triple::Contiki; } + /// Tests whether the OS is Haiku. + bool isOSHaiku() const { + return getOS() == Triple::Haiku; + } + /// Checks if the environment could be MSVC. bool isWindowsMSVCEnvironment() const { return getOS() == Triple::Win32 && @@ -634,8 +645,25 @@ public: return getArch() == Triple::nvptx || getArch() == Triple::nvptx64; } + /// Tests whether the target is Thumb (little and big endian). + bool isThumb() const { + return getArch() == Triple::thumb || getArch() == Triple::thumbeb; + } + + /// Tests whether the target is ARM (little and big endian). + bool isARM() const { + return getArch() == Triple::arm || getArch() == Triple::armeb; + } + + /// Tests whether the target is AArch64 (little and big endian). + bool isAArch64() const { + return getArch() == Triple::aarch64 || getArch() == Triple::aarch64_be; + } + /// Tests wether the target supports comdat - bool supportsCOMDAT() const { return !isOSBinFormatMachO(); } + bool supportsCOMDAT() const { + return !isOSBinFormatMachO() && !isOSBinFormatWasm(); + } /// @} /// @name Mutators diff --git a/contrib/llvm/include/llvm/ADT/Twine.h b/contrib/llvm/include/llvm/ADT/Twine.h index f5f00dcfafe5..b60fd0981398 100644 --- a/contrib/llvm/include/llvm/ADT/Twine.h +++ b/contrib/llvm/include/llvm/ADT/Twine.h @@ -1,4 +1,4 @@ -//===-- Twine.h - Fast Temporary String Concatenation -----------*- C++ -*-===// +//===- Twine.h - Fast Temporary String Concatenation ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -155,17 +155,19 @@ namespace llvm { /// LHS - The prefix in the concatenation, which may be uninitialized for /// Null or Empty kinds. Child LHS; + /// RHS - The suffix in the concatenation, which may be uninitialized for /// Null or Empty kinds. Child RHS; + /// LHSKind - The NodeKind of the left hand side, \see getLHSKind(). - NodeKind LHSKind; + NodeKind LHSKind = EmptyKind; + /// RHSKind - The NodeKind of the right hand side, \see getRHSKind(). - NodeKind RHSKind; + NodeKind RHSKind = EmptyKind; /// Construct a nullary twine; the kind must be NullKind or EmptyKind. - explicit Twine(NodeKind Kind) - : LHSKind(Kind), RHSKind(EmptyKind) { + explicit Twine(NodeKind Kind) : LHSKind(Kind) { assert(isNullary() && "Invalid kind!"); } @@ -252,7 +254,7 @@ namespace llvm { /// @{ /// Construct from an empty string. - /*implicit*/ Twine() : LHSKind(EmptyKind), RHSKind(EmptyKind) { + /*implicit*/ Twine() { assert(isValid() && "Invalid twine!"); } @@ -263,8 +265,7 @@ namespace llvm { /// We take care here to optimize "" into the empty twine -- this will be /// optimized out for string constants. This allows Twine arguments have /// default "" values, without introducing unnecessary string constants. - /*implicit*/ Twine(const char *Str) - : RHSKind(EmptyKind) { + /*implicit*/ Twine(const char *Str) { if (Str[0] != '\0') { LHS.cString = Str; LHSKind = CStringKind; @@ -275,84 +276,73 @@ namespace llvm { } /// Construct from an std::string. - /*implicit*/ Twine(const std::string &Str) - : LHSKind(StdStringKind), RHSKind(EmptyKind) { + /*implicit*/ Twine(const std::string &Str) : LHSKind(StdStringKind) { LHS.stdString = &Str; assert(isValid() && "Invalid twine!"); } /// Construct from a StringRef. - /*implicit*/ Twine(const StringRef &Str) - : LHSKind(StringRefKind), RHSKind(EmptyKind) { + /*implicit*/ Twine(const StringRef &Str) : LHSKind(StringRefKind) { LHS.stringRef = &Str; assert(isValid() && "Invalid twine!"); } /// Construct from a SmallString. /*implicit*/ Twine(const SmallVectorImpl<char> &Str) - : LHSKind(SmallStringKind), RHSKind(EmptyKind) { + : LHSKind(SmallStringKind) { LHS.smallString = &Str; assert(isValid() && "Invalid twine!"); } /// Construct from a formatv_object_base. /*implicit*/ Twine(const formatv_object_base &Fmt) - : LHSKind(FormatvObjectKind), RHSKind(EmptyKind) { + : LHSKind(FormatvObjectKind) { LHS.formatvObject = &Fmt; assert(isValid() && "Invalid twine!"); } /// Construct from a char. - explicit Twine(char Val) - : LHSKind(CharKind), RHSKind(EmptyKind) { + explicit Twine(char Val) : LHSKind(CharKind) { LHS.character = Val; } /// Construct from a signed char. - explicit Twine(signed char Val) - : LHSKind(CharKind), RHSKind(EmptyKind) { + explicit Twine(signed char Val) : LHSKind(CharKind) { LHS.character = static_cast<char>(Val); } /// Construct from an unsigned char. - explicit Twine(unsigned char Val) - : LHSKind(CharKind), RHSKind(EmptyKind) { + explicit Twine(unsigned char Val) : LHSKind(CharKind) { LHS.character = static_cast<char>(Val); } /// Construct a twine to print \p Val as an unsigned decimal integer. - explicit Twine(unsigned Val) - : LHSKind(DecUIKind), RHSKind(EmptyKind) { + explicit Twine(unsigned Val) : LHSKind(DecUIKind) { LHS.decUI = Val; } /// Construct a twine to print \p Val as a signed decimal integer. - explicit Twine(int Val) - : LHSKind(DecIKind), RHSKind(EmptyKind) { + explicit Twine(int Val) : LHSKind(DecIKind) { LHS.decI = Val; } /// Construct a twine to print \p Val as an unsigned decimal integer. - explicit Twine(const unsigned long &Val) - : LHSKind(DecULKind), RHSKind(EmptyKind) { + explicit Twine(const unsigned long &Val) : LHSKind(DecULKind) { LHS.decUL = &Val; } /// Construct a twine to print \p Val as a signed decimal integer. - explicit Twine(const long &Val) - : LHSKind(DecLKind), RHSKind(EmptyKind) { + explicit Twine(const long &Val) : LHSKind(DecLKind) { LHS.decL = &Val; } /// Construct a twine to print \p Val as an unsigned decimal integer. - explicit Twine(const unsigned long long &Val) - : LHSKind(DecULLKind), RHSKind(EmptyKind) { + explicit Twine(const unsigned long long &Val) : LHSKind(DecULLKind) { LHS.decULL = &Val; } /// Construct a twine to print \p Val as a signed decimal integer. - explicit Twine(const long long &Val) - : LHSKind(DecLLKind), RHSKind(EmptyKind) { + explicit Twine(const long long &Val) : LHSKind(DecLLKind) { LHS.decLL = &Val; } diff --git a/contrib/llvm/include/llvm/ADT/iterator.h b/contrib/llvm/include/llvm/ADT/iterator.h index 15720a67c047..711f8f221620 100644 --- a/contrib/llvm/include/llvm/ADT/iterator.h +++ b/contrib/llvm/include/llvm/ADT/iterator.h @@ -70,10 +70,10 @@ class iterator_facade_base ReferenceT> { protected: enum { - IsRandomAccess = - std::is_base_of<std::random_access_iterator_tag, IteratorCategoryT>::value, - IsBidirectional = - std::is_base_of<std::bidirectional_iterator_tag, IteratorCategoryT>::value, + IsRandomAccess = std::is_base_of<std::random_access_iterator_tag, + IteratorCategoryT>::value, + IsBidirectional = std::is_base_of<std::bidirectional_iterator_tag, + IteratorCategoryT>::value, }; /// A proxy object for computing a reference via indirecting a copy of an diff --git a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h index e00ae4f3beec..9de075dfd681 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -38,24 +38,30 @@ #ifndef LLVM_ANALYSIS_ALIASANALYSIS_H #define LLVM_ANALYSIS_ALIASANALYSIS_H +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CallSite.h" -#include "llvm/IR/Metadata.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" +#include <cstdint> +#include <functional> +#include <memory> +#include <vector> namespace llvm { -class BasicAAResult; -class LoadInst; -class StoreInst; -class VAArgInst; -class DataLayout; -class Pass; + class AnalysisUsage; -class MemTransferInst; -class MemIntrinsic; +class BasicAAResult; +class BasicBlock; class DominatorTree; class OrderedBasicBlock; +class Value; /// The possible results of an alias query. /// @@ -89,19 +95,62 @@ enum AliasResult { /// /// This is no access at all, a modification, a reference, or both /// a modification and a reference. These are specifically structured such that -/// they form a two bit matrix and bit-tests for 'mod' or 'ref' work with any -/// of the possible values. -enum ModRefInfo { +/// they form a two bit matrix and bit-tests for 'mod' or 'ref' +/// work with any of the possible values. + +enum class ModRefInfo { /// The access neither references nor modifies the value stored in memory. - MRI_NoModRef = 0, - /// The access references the value stored in memory. - MRI_Ref = 1, - /// The access modifies the value stored in memory. - MRI_Mod = 2, - /// The access both references and modifies the value stored in memory. - MRI_ModRef = MRI_Ref | MRI_Mod + NoModRef = 0, + /// The access may reference the value stored in memory. + Ref = 1, + /// The access may modify the value stored in memory. + Mod = 2, + /// The access may reference and may modify the value stored in memory. + ModRef = Ref | Mod, }; +LLVM_NODISCARD inline bool isNoModRef(const ModRefInfo MRI) { + return MRI == ModRefInfo::NoModRef; +} +LLVM_NODISCARD inline bool isModOrRefSet(const ModRefInfo MRI) { + return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::ModRef); +} +LLVM_NODISCARD inline bool isModAndRefSet(const ModRefInfo MRI) { + return (static_cast<int>(MRI) & static_cast<int>(ModRefInfo::ModRef)) == + static_cast<int>(ModRefInfo::ModRef); +} +LLVM_NODISCARD inline bool isModSet(const ModRefInfo MRI) { + return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Mod); +} +LLVM_NODISCARD inline bool isRefSet(const ModRefInfo MRI) { + return static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Ref); +} + +LLVM_NODISCARD inline ModRefInfo setMod(const ModRefInfo MRI) { + return ModRefInfo(static_cast<int>(MRI) | static_cast<int>(ModRefInfo::Mod)); +} +LLVM_NODISCARD inline ModRefInfo setRef(const ModRefInfo MRI) { + return ModRefInfo(static_cast<int>(MRI) | static_cast<int>(ModRefInfo::Ref)); +} +LLVM_NODISCARD inline ModRefInfo setModAndRef(const ModRefInfo MRI) { + return ModRefInfo(static_cast<int>(MRI) | + static_cast<int>(ModRefInfo::ModRef)); +} +LLVM_NODISCARD inline ModRefInfo clearMod(const ModRefInfo MRI) { + return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Ref)); +} +LLVM_NODISCARD inline ModRefInfo clearRef(const ModRefInfo MRI) { + return ModRefInfo(static_cast<int>(MRI) & static_cast<int>(ModRefInfo::Mod)); +} +LLVM_NODISCARD inline ModRefInfo unionModRef(const ModRefInfo MRI1, + const ModRefInfo MRI2) { + return ModRefInfo(static_cast<int>(MRI1) | static_cast<int>(MRI2)); +} +LLVM_NODISCARD inline ModRefInfo intersectModRef(const ModRefInfo MRI1, + const ModRefInfo MRI2) { + return ModRefInfo(static_cast<int>(MRI1) & static_cast<int>(MRI2)); +} + /// The locations at which a function might access memory. /// /// These are primarily used in conjunction with the \c AccessKind bits to @@ -129,27 +178,31 @@ enum FunctionModRefBehavior { /// This property corresponds to the GCC 'const' attribute. /// This property corresponds to the LLVM IR 'readnone' attribute. /// This property corresponds to the IntrNoMem LLVM intrinsic flag. - FMRB_DoesNotAccessMemory = FMRL_Nowhere | MRI_NoModRef, + FMRB_DoesNotAccessMemory = + FMRL_Nowhere | static_cast<int>(ModRefInfo::NoModRef), /// The only memory references in this function (if it has any) are /// non-volatile loads from objects pointed to by its pointer-typed /// arguments, with arbitrary offsets. /// /// This property corresponds to the IntrReadArgMem LLVM intrinsic flag. - FMRB_OnlyReadsArgumentPointees = FMRL_ArgumentPointees | MRI_Ref, + FMRB_OnlyReadsArgumentPointees = + FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::Ref), /// The only memory references in this function (if it has any) are /// non-volatile loads and stores from objects pointed to by its /// pointer-typed arguments, with arbitrary offsets. /// /// This property corresponds to the IntrArgMemOnly LLVM intrinsic flag. - FMRB_OnlyAccessesArgumentPointees = FMRL_ArgumentPointees | MRI_ModRef, + FMRB_OnlyAccessesArgumentPointees = + FMRL_ArgumentPointees | static_cast<int>(ModRefInfo::ModRef), /// The only memory references in this function (if it has any) are /// references of memory that is otherwise inaccessible via LLVM IR. /// /// This property corresponds to the LLVM IR inaccessiblememonly attribute. - FMRB_OnlyAccessesInaccessibleMem = FMRL_InaccessibleMem | MRI_ModRef, + FMRB_OnlyAccessesInaccessibleMem = + FMRL_InaccessibleMem | static_cast<int>(ModRefInfo::ModRef), /// The function may perform non-volatile loads and stores of objects /// pointed to by its pointer-typed arguments, with arbitrary offsets, and @@ -159,7 +212,8 @@ enum FunctionModRefBehavior { /// This property corresponds to the LLVM IR /// inaccessiblemem_or_argmemonly attribute. FMRB_OnlyAccessesInaccessibleOrArgMem = FMRL_InaccessibleMem | - FMRL_ArgumentPointees | MRI_ModRef, + FMRL_ArgumentPointees | + static_cast<int>(ModRefInfo::ModRef), /// This function does not perform any non-local stores or volatile loads, /// but may read from any memory location. @@ -167,20 +221,30 @@ enum FunctionModRefBehavior { /// This property corresponds to the GCC 'pure' attribute. /// This property corresponds to the LLVM IR 'readonly' attribute. /// This property corresponds to the IntrReadMem LLVM intrinsic flag. - FMRB_OnlyReadsMemory = FMRL_Anywhere | MRI_Ref, + FMRB_OnlyReadsMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Ref), // This function does not read from memory anywhere, but may write to any // memory location. // // This property corresponds to the LLVM IR 'writeonly' attribute. // This property corresponds to the IntrWriteMem LLVM intrinsic flag. - FMRB_DoesNotReadMemory = FMRL_Anywhere | MRI_Mod, + FMRB_DoesNotReadMemory = FMRL_Anywhere | static_cast<int>(ModRefInfo::Mod), /// This indicates that the function could not be classified into one of the /// behaviors above. - FMRB_UnknownModRefBehavior = FMRL_Anywhere | MRI_ModRef + FMRB_UnknownModRefBehavior = + FMRL_Anywhere | static_cast<int>(ModRefInfo::ModRef) }; +// Wrapper method strips bits significant only in FunctionModRefBehavior, +// to obtain a valid ModRefInfo. The benefit of using the wrapper is that if +// ModRefInfo enum changes, the wrapper can be updated to & with the new enum +// entry with all bits set to 1. +LLVM_NODISCARD inline ModRefInfo +createModRefInfo(const FunctionModRefBehavior FMRB) { + return ModRefInfo(FMRB & static_cast<int>(ModRefInfo::ModRef)); +} + class AAResults { public: // Make these results default constructable and movable. We have to spell @@ -348,13 +412,13 @@ public: /// Checks if functions with the specified behavior are known to only read /// from non-volatile memory (or not access memory at all). static bool onlyReadsMemory(FunctionModRefBehavior MRB) { - return !(MRB & MRI_Mod); + return !isModSet(createModRefInfo(MRB)); } /// Checks if functions with the specified behavior are known to only write /// memory (or not access memory at all). static bool doesNotReadMemory(FunctionModRefBehavior MRB) { - return !(MRB & MRI_Ref); + return !isRefSet(createModRefInfo(MRB)); } /// Checks if functions with the specified behavior are known to read and @@ -368,7 +432,8 @@ public: /// read or write from objects pointed to be their pointer-typed arguments /// (with arbitrary offsets). static bool doesAccessArgPointees(FunctionModRefBehavior MRB) { - return (MRB & MRI_ModRef) && (MRB & FMRL_ArgumentPointees); + return isModOrRefSet(createModRefInfo(MRB)) && + (MRB & FMRL_ArgumentPointees); } /// Checks if functions with the specified behavior are known to read and @@ -380,7 +445,7 @@ public: /// Checks if functions with the specified behavior are known to potentially /// read or write from memory that is inaccessible from LLVM IR. static bool doesAccessInaccessibleMem(FunctionModRefBehavior MRB) { - return (MRB & MRI_ModRef) && (MRB & FMRL_InaccessibleMem); + return isModOrRefSet(createModRefInfo(MRB)) && (MRB & FMRL_InaccessibleMem); } /// Checks if functions with the specified behavior are known to read and @@ -500,43 +565,26 @@ public: return getModRefInfo(I, MemoryLocation(P, Size)); } - /// Check whether or not an instruction may read or write memory (without - /// regard to a specific location). + /// Check whether or not an instruction may read or write the optionally + /// specified memory location. /// - /// For function calls, this delegates to the alias-analysis specific - /// call-site mod-ref behavior queries. Otherwise it delegates to the generic - /// mod ref information query without a location. - ModRefInfo getModRefInfo(const Instruction *I) { - if (auto CS = ImmutableCallSite(I)) { - auto MRB = getModRefBehavior(CS); - if ((MRB & MRI_ModRef) == MRI_ModRef) - return MRI_ModRef; - if (MRB & MRI_Ref) - return MRI_Ref; - if (MRB & MRI_Mod) - return MRI_Mod; - return MRI_NoModRef; - } - - return getModRefInfo(I, MemoryLocation()); - } - - /// Check whether or not an instruction may read or write the specified - /// memory location. - /// - /// Note explicitly that getModRefInfo considers the effects of reading and - /// writing the memory location, and not the effect of ordering relative to - /// other instructions. Thus, a volatile load is considered to be Ref, - /// because it does not actually write memory, it just can't be reordered - /// relative to other volatiles (or removed). Atomic ordered loads/stores are - /// considered ModRef ATM because conservatively, the visible effect appears - /// as if memory was written, not just an ordering constraint. /// /// An instruction that doesn't read or write memory may be trivially LICM'd /// for example. /// - /// This primarily delegates to specific helpers above. - ModRefInfo getModRefInfo(const Instruction *I, const MemoryLocation &Loc) { + /// For function calls, this delegates to the alias-analysis specific + /// call-site mod-ref behavior queries. Otherwise it delegates to the specific + /// helpers above. + ModRefInfo getModRefInfo(const Instruction *I, + const Optional<MemoryLocation> &OptLoc) { + if (OptLoc == None) { + if (auto CS = ImmutableCallSite(I)) { + return createModRefInfo(getModRefBehavior(CS)); + } + } + + const MemoryLocation &Loc = OptLoc.getValueOr(MemoryLocation()); + switch (I->getOpcode()) { case Instruction::VAArg: return getModRefInfo((const VAArgInst*)I, Loc); case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc); @@ -553,7 +601,7 @@ public: case Instruction::CatchRet: return getModRefInfo((const CatchReturnInst *)I, Loc); default: - return MRI_NoModRef; + return ModRefInfo::NoModRef; } } @@ -574,7 +622,7 @@ public: /// \brief Return information about whether a particular call site modifies /// or reads the specified memory location \p MemLoc before instruction \p I - /// in a BasicBlock. A ordered basic block \p OBB can be used to speed up + /// in a BasicBlock. An ordered basic block \p OBB can be used to speed up /// instruction ordering queries inside the BasicBlock containing \p I. ModRefInfo callCapturesBefore(const Instruction *I, const MemoryLocation &MemLoc, DominatorTree *DT, @@ -620,6 +668,7 @@ public: private: class Concept; + template <typename T> class Model; template <typename T> friend class AAResultBase; @@ -633,7 +682,7 @@ private: /// Temporary typedef for legacy code that uses a generic \c AliasAnalysis /// pointer or reference. -typedef AAResults AliasAnalysis; +using AliasAnalysis = AAResults; /// A private abstract base class describing the concept of an individual alias /// analysis implementation. @@ -714,7 +763,7 @@ public: explicit Model(AAResultT &Result, AAResults &AAR) : Result(Result) { Result.setAAResults(&AAR); } - ~Model() override {} + ~Model() override = default; void setAAResults(AAResults *NewAAR) override { Result.setAAResults(NewAAR); } @@ -824,7 +873,7 @@ protected: } }; - explicit AAResultBase() {} + explicit AAResultBase() = default; // Provide all the copy and move constructors so that derived types aren't // constrained. @@ -853,7 +902,7 @@ public: } ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) { - return MRI_ModRef; + return ModRefInfo::ModRef; } FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) { @@ -865,15 +914,14 @@ public: } ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) { - return MRI_ModRef; + return ModRefInfo::ModRef; } ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) { - return MRI_ModRef; + return ModRefInfo::ModRef; } }; - /// Return true if this pointer is returned by a noalias function. bool isNoAliasCall(const Value *V); @@ -910,7 +958,7 @@ bool isIdentifiedFunctionLocal(const Value *V); /// ensure the analysis itself is registered with its AnalysisManager. class AAManager : public AnalysisInfoMixin<AAManager> { public: - typedef AAResults Result; + using Result = AAResults; /// Register a specific AA result. template <typename AnalysisT> void registerFunctionAnalysis() { @@ -931,6 +979,7 @@ public: private: friend AnalysisInfoMixin<AAManager>; + static AnalysisKey Key; SmallVector<void (*)(Function &F, FunctionAnalysisManager &AM, @@ -1001,6 +1050,6 @@ AAResults createLegacyPMAAResults(Pass &P, Function &F, BasicAAResult &BAR); /// sure the analyses required by \p createLegacyPMAAResults are available. void getAAResultsAnalysisUsage(AnalysisUsage &AU); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_ALIASANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h index daafd2fabe78..7da3ebabb8a3 100644 --- a/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h +++ b/contrib/llvm/include/llvm/Analysis/AliasSetTracker.h @@ -18,36 +18,46 @@ #define LLVM_ANALYSIS_ALIASSETTRACKER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Casting.h" +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <iterator> #include <vector> namespace llvm { +class AliasSetTracker; +class BasicBlock; class LoadInst; +class MemSetInst; +class MemTransferInst; +class raw_ostream; class StoreInst; class VAArgInst; -class MemSetInst; -class AliasSetTracker; -class AliasSet; +class Value; class AliasSet : public ilist_node<AliasSet> { friend class AliasSetTracker; class PointerRec { Value *Val; // The pointer this record corresponds to. - PointerRec **PrevInList, *NextInList; - AliasSet *AS; - uint64_t Size; + PointerRec **PrevInList = nullptr; + PointerRec *NextInList = nullptr; + AliasSet *AS = nullptr; + uint64_t Size = 0; AAMDNodes AAInfo; public: PointerRec(Value *V) - : Val(V), PrevInList(nullptr), NextInList(nullptr), AS(nullptr), Size(0), - AAInfo(DenseMapInfo<AAMDNodes>::getEmptyKey()) {} + : Val(V), AAInfo(DenseMapInfo<AAMDNodes>::getEmptyKey()) {} Value *getValue() const { return Val; } @@ -121,9 +131,10 @@ class AliasSet : public ilist_node<AliasSet> { }; // Doubly linked list of nodes. - PointerRec *PtrList, **PtrListEnd; + PointerRec *PtrList = nullptr; + PointerRec **PtrListEnd; // Forwarding pointer. - AliasSet *Forward; + AliasSet *Forward = nullptr; /// All instructions without a specific address in this alias set. /// In rare cases this vector can have a null'ed out WeakVH @@ -167,7 +178,7 @@ class AliasSet : public ilist_node<AliasSet> { /// True if this alias set contains volatile loads or stores. unsigned Volatile : 1; - unsigned SetSize; + unsigned SetSize = 0; void addRef() { ++RefCount; } @@ -183,6 +194,9 @@ class AliasSet : public ilist_node<AliasSet> { } public: + AliasSet(const AliasSet &) = delete; + AliasSet &operator=(const AliasSet &) = delete; + /// Accessors... bool isRef() const { return Access & RefAccess; } bool isMod() const { return Access & ModAccess; } @@ -249,12 +263,8 @@ public: private: // Can only be created by AliasSetTracker. AliasSet() - : PtrList(nullptr), PtrListEnd(&PtrList), Forward(nullptr), RefCount(0), - AliasAny(false), Access(NoAccess), Alias(SetMustAlias), - Volatile(false), SetSize(0) {} - - AliasSet(const AliasSet &AS) = delete; - void operator=(const AliasSet &AS) = delete; + : PtrListEnd(&PtrList), RefCount(0), AliasAny(false), Access(NoAccess), + Alias(SetMustAlias), Volatile(false) {} PointerRec *getSomePointer() const { return PtrList; @@ -281,6 +291,7 @@ private: const AAMDNodes &AAInfo, bool KnownMustAlias = false); void addUnknownInst(Instruction *I, AliasAnalysis &AA); + void removeUnknownInst(AliasSetTracker &AST, Instruction *I) { bool WasEmpty = UnknownInsts.empty(); for (size_t i = 0, e = UnknownInsts.size(); i != e; ++i) @@ -292,6 +303,7 @@ private: if (!WasEmpty && UnknownInsts.empty()) dropRef(AST); } + void setVolatile() { Volatile = true; } public: @@ -312,11 +324,13 @@ class AliasSetTracker { /// Value is deleted. class ASTCallbackVH final : public CallbackVH { AliasSetTracker *AST; + void deleted() override; void allUsesReplacedWith(Value *) override; public: ASTCallbackVH(Value *V, AliasSetTracker *AST = nullptr); + ASTCallbackVH &operator=(Value *V); }; /// Traits to tell DenseMap that tell us how to compare and hash the value @@ -326,9 +340,8 @@ class AliasSetTracker { AliasAnalysis &AA; ilist<AliasSet> AliasSets; - typedef DenseMap<ASTCallbackVH, AliasSet::PointerRec*, - ASTCallbackVHDenseMapInfo> - PointerMapType; + using PointerMapType = DenseMap<ASTCallbackVH, AliasSet::PointerRec *, + ASTCallbackVHDenseMapInfo>; // Map from pointers to their node PointerMapType PointerMap; @@ -336,8 +349,7 @@ class AliasSetTracker { public: /// Create an empty collection of AliasSets, and use the specified alias /// analysis object to disambiguate load and store addresses. - explicit AliasSetTracker(AliasAnalysis &aa) - : AA(aa), TotalMayAliasSetSize(0), AliasAnyAS(nullptr) {} + explicit AliasSetTracker(AliasAnalysis &aa) : AA(aa) {} ~AliasSetTracker() { clear(); } /// These methods are used to add different types of instructions to the alias @@ -401,8 +413,8 @@ public: /// tracker already knows about a value, it will ignore the request. void copyValue(Value *From, Value *To); - typedef ilist<AliasSet>::iterator iterator; - typedef ilist<AliasSet>::const_iterator const_iterator; + using iterator = ilist<AliasSet>::iterator; + using const_iterator = ilist<AliasSet>::const_iterator; const_iterator begin() const { return AliasSets.begin(); } const_iterator end() const { return AliasSets.end(); } @@ -417,11 +429,11 @@ private: friend class AliasSet; // The total number of pointers contained in all "may" alias sets. - unsigned TotalMayAliasSetSize; + unsigned TotalMayAliasSetSize = 0; // A non-null value signifies this AST is saturated. A saturated AST lumps // all pointers into a single "May" set. - AliasSet *AliasAnyAS; + AliasSet *AliasAnyAS = nullptr; void removeAliasSet(AliasSet *AS); @@ -451,6 +463,6 @@ inline raw_ostream& operator<<(raw_ostream &OS, const AliasSetTracker &AST) { return OS; } -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_ALIASSETTRACKER_H diff --git a/contrib/llvm/include/llvm/Analysis/AssumptionCache.h b/contrib/llvm/include/llvm/Analysis/AssumptionCache.h index 58d72afdc1b6..c965e62a0216 100644 --- a/contrib/llvm/include/llvm/Analysis/AssumptionCache.h +++ b/contrib/llvm/include/llvm/Analysis/AssumptionCache.h @@ -1,4 +1,4 @@ -//===- llvm/Analysis/AssumptionCache.h - Track @llvm.assume ---*- C++ -*-===// +//===- llvm/Analysis/AssumptionCache.h - Track @llvm.assume -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -18,9 +18,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" @@ -28,6 +27,11 @@ namespace llvm { +class CallInst; +class Function; +class raw_ostream; +class Value; + /// \brief A cache of @llvm.assume calls within a function. /// /// This cache provides fast lookup of assumptions within a function by caching @@ -47,6 +51,7 @@ class AssumptionCache { class AffectedValueCallbackVH final : public CallbackVH { AssumptionCache *AC; + void deleted() override; void allUsesReplacedWith(Value *) override; @@ -76,7 +81,7 @@ class AssumptionCache { /// /// We want to be as lazy about this as possible, and so we scan the function /// at the last moment. - bool Scanned; + bool Scanned = false; /// \brief Scan the function for assumptions and add them to the cache. void scanFunction(); @@ -84,7 +89,7 @@ class AssumptionCache { public: /// \brief Construct an AssumptionCache from a function by scanning all of /// its instructions. - AssumptionCache(Function &F) : F(F), Scanned(false) {} + AssumptionCache(Function &F) : F(F) {} /// This cache is designed to be self-updating and so it should never be /// invalidated. @@ -145,10 +150,11 @@ public: /// assumption caches for a given function. class AssumptionAnalysis : public AnalysisInfoMixin<AssumptionAnalysis> { friend AnalysisInfoMixin<AssumptionAnalysis>; + static AnalysisKey Key; public: - typedef AssumptionCache Result; + using Result = AssumptionCache; AssumptionCache run(Function &F, FunctionAnalysisManager &) { return AssumptionCache(F); @@ -161,6 +167,7 @@ class AssumptionPrinterPass : public PassInfoMixin<AssumptionPrinterPass> { public: explicit AssumptionPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; @@ -177,10 +184,11 @@ class AssumptionCacheTracker : public ImmutablePass { /// delete our cache of intrinsics for a function when it is deleted. class FunctionCallbackVH final : public CallbackVH { AssumptionCacheTracker *ACT; + void deleted() override; public: - typedef DenseMapInfo<Value *> DMI; + using DMI = DenseMapInfo<Value *>; FunctionCallbackVH(Value *V, AssumptionCacheTracker *ACT = nullptr) : CallbackVH(V), ACT(ACT) {} @@ -188,8 +196,10 @@ class AssumptionCacheTracker : public ImmutablePass { friend FunctionCallbackVH; - typedef DenseMap<FunctionCallbackVH, std::unique_ptr<AssumptionCache>, - FunctionCallbackVH::DMI> FunctionCallsMap; + using FunctionCallsMap = + DenseMap<FunctionCallbackVH, std::unique_ptr<AssumptionCache>, + FunctionCallbackVH::DMI>; + FunctionCallsMap AssumptionCaches; public: @@ -208,6 +218,7 @@ public: } void verifyAnalysis() const override; + bool doFinalization(Module &) override { verifyAnalysis(); return false; @@ -218,4 +229,4 @@ public: } // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_ASSUMPTIONCACHE_H diff --git a/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h index 14e4bded264a..42e5e9714071 100644 --- a/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/BasicAliasAnalysis.h @@ -14,22 +14,36 @@ #ifndef LLVM_ANALYSIS_BASICALIASANALYSIS_H #define LLVM_ANALYSIS_BASICALIASANALYSIS_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" -#include "llvm/IR/Instruction.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" +#include "llvm/Analysis/MemoryLocation.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/ErrorHandling.h" +#include "llvm/Pass.h" +#include <algorithm> +#include <cstdint> +#include <memory> +#include <utility> namespace llvm { + +struct AAMDNodes; +class APInt; class AssumptionCache; +class BasicBlock; +class DataLayout; class DominatorTree; +class Function; +class GEPOperator; class LoopInfo; +class PHINode; +class SelectInst; +class TargetLibraryInfo; +class Value; /// This is the AA result object for the basic, local, and stateless alias /// analysis. It implements the AA query interface in an entirely stateless @@ -86,7 +100,6 @@ private: // A linear transformation of a Value; this class represents ZExt(SExt(V, // SExtBits), ZExtBits) * Scale + Offset. struct VariableGEPIndex { - // An opaque Value - we can't decompose this further. const Value *V; @@ -124,8 +137,8 @@ private: }; /// Track alias queries to guard against recursion. - typedef std::pair<MemoryLocation, MemoryLocation> LocPair; - typedef SmallDenseMap<LocPair, AliasResult, 8> AliasCacheTy; + using LocPair = std::pair<MemoryLocation, MemoryLocation>; + using AliasCacheTy = SmallDenseMap<LocPair, AliasResult, 8>; AliasCacheTy AliasCache; /// Tracks phi nodes we have visited. @@ -201,10 +214,11 @@ private: /// Analysis pass providing a never-invalidated alias analysis result. class BasicAA : public AnalysisInfoMixin<BasicAA> { friend AnalysisInfoMixin<BasicAA>; + static AnalysisKey Key; public: - typedef BasicAAResult Result; + using Result = BasicAAResult; BasicAAResult run(Function &F, FunctionAnalysisManager &AM); }; @@ -251,6 +265,6 @@ public: } }; -} +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_BASICALIASANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h index cbae01c9102f..89370cbeeea1 100644 --- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h +++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfo.h @@ -18,31 +18,36 @@ #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/BlockFrequency.h" -#include <climits> +#include <cstdint> +#include <memory> namespace llvm { +class BasicBlock; class BranchProbabilityInfo; +class Function; class LoopInfo; +class Module; +class raw_ostream; template <class BlockT> class BlockFrequencyInfoImpl; +enum PGOViewCountsType { PGOVCT_None, PGOVCT_Graph, PGOVCT_Text }; + /// BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to /// estimate IR basic block frequencies. class BlockFrequencyInfo { - typedef BlockFrequencyInfoImpl<BasicBlock> ImplType; - std::unique_ptr<ImplType> BFI; + using ImplType = BlockFrequencyInfoImpl<BasicBlock>; - void operator=(const BlockFrequencyInfo &) = delete; - BlockFrequencyInfo(const BlockFrequencyInfo &) = delete; + std::unique_ptr<ImplType> BFI; public: BlockFrequencyInfo(); BlockFrequencyInfo(const Function &F, const BranchProbabilityInfo &BPI, const LoopInfo &LI); + BlockFrequencyInfo(const BlockFrequencyInfo &) = delete; + BlockFrequencyInfo &operator=(const BlockFrequencyInfo &) = delete; BlockFrequencyInfo(BlockFrequencyInfo &&Arg); - BlockFrequencyInfo &operator=(BlockFrequencyInfo &&RHS); - ~BlockFrequencyInfo(); /// Handle invalidation explicitly. @@ -70,6 +75,10 @@ public: /// the enclosing function's count (if available) and returns the value. Optional<uint64_t> getProfileCountFromFreq(uint64_t Freq) const; + /// \brief Returns true if \p BB is an irreducible loop header + /// block. Otherwise false. + bool isIrrLoopHeader(const BasicBlock *BB); + // Set the frequency of the given basic block. void setBlockFreq(const BasicBlock *BB, uint64_t Freq); @@ -100,11 +109,12 @@ public: class BlockFrequencyAnalysis : public AnalysisInfoMixin<BlockFrequencyAnalysis> { friend AnalysisInfoMixin<BlockFrequencyAnalysis>; + static AnalysisKey Key; public: - /// \brief Provide the result typedef for this analysis pass. - typedef BlockFrequencyInfo Result; + /// \brief Provide the result type for this analysis pass. + using Result = BlockFrequencyInfo; /// \brief Run the analysis pass over a function and produce BFI. Result run(Function &F, FunctionAnalysisManager &AM); @@ -117,6 +127,7 @@ class BlockFrequencyPrinterPass public: explicit BlockFrequencyPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; @@ -140,6 +151,6 @@ public: void print(raw_ostream &OS, const Module *M) const override; }; -} +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_BLOCKFREQUENCYINFO_H diff --git a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 5de3821242e0..40c40b80bc89 100644 --- a/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -1,4 +1,4 @@ -//==- BlockFrequencyInfoImpl.h - Block Frequency Implementation -*- C++ -*-===// +//==- BlockFrequencyInfoImpl.h - Block Frequency Implementation --*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -16,28 +16,39 @@ #define LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SparseBitVector.h" +#include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/BasicBlock.h" #include "llvm/Support/BlockFrequency.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/ScaledNumber.h" #include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> #include <deque> +#include <iterator> +#include <limits> #include <list> #include <string> +#include <utility> #include <vector> #define DEBUG_TYPE "block-freq" namespace llvm { -class BasicBlock; class BranchProbabilityInfo; class Function; class Loop; @@ -58,7 +69,8 @@ template <class BT> struct BlockEdgesAdder; /// \brief Mass of a block. /// /// This class implements a sort of fixed-point fraction always between 0.0 and -/// 1.0. getMass() == UINT64_MAX indicates a value of 1.0. +/// 1.0. getMass() == std::numeric_limits<uint64_t>::max() indicates a value of +/// 1.0. /// /// Masses can be added and subtracted. Simple saturation arithmetic is used, /// so arithmetic operations never overflow or underflow. @@ -69,18 +81,21 @@ template <class BT> struct BlockEdgesAdder; /// /// Masses can be scaled by \a BranchProbability at maximum precision. class BlockMass { - uint64_t Mass; + uint64_t Mass = 0; public: - BlockMass() : Mass(0) {} + BlockMass() = default; explicit BlockMass(uint64_t Mass) : Mass(Mass) {} static BlockMass getEmpty() { return BlockMass(); } - static BlockMass getFull() { return BlockMass(UINT64_MAX); } + + static BlockMass getFull() { + return BlockMass(std::numeric_limits<uint64_t>::max()); + } uint64_t getMass() const { return Mass; } - bool isFull() const { return Mass == UINT64_MAX; } + bool isFull() const { return Mass == std::numeric_limits<uint64_t>::max(); } bool isEmpty() const { return !Mass; } bool operator!() const { return isEmpty(); } @@ -90,7 +105,7 @@ public: /// Adds another mass, saturating at \a isFull() rather than overflowing. BlockMass &operator+=(BlockMass X) { uint64_t Sum = Mass + X.Mass; - Mass = Sum < Mass ? UINT64_MAX : Sum; + Mass = Sum < Mass ? std::numeric_limits<uint64_t>::max() : Sum; return *this; } @@ -159,8 +174,8 @@ template <> struct isPodLike<bfi_detail::BlockMass> { /// BlockFrequencyInfoImpl. See there for details. class BlockFrequencyInfoImplBase { public: - typedef ScaledNumber<uint64_t> Scaled64; - typedef bfi_detail::BlockMass BlockMass; + using Scaled64 = ScaledNumber<uint64_t>; + using BlockMass = bfi_detail::BlockMass; /// \brief Representative of a block. /// @@ -170,8 +185,12 @@ public: /// Unlike a block pointer, its order has meaning (location in the /// topological sort) and it's class is the same regardless of block type. struct BlockNode { - typedef uint32_t IndexType; - IndexType Index; + using IndexType = uint32_t; + + IndexType Index = std::numeric_limits<uint32_t>::max(); + + BlockNode() = default; + BlockNode(IndexType Index) : Index(Index) {} bool operator==(const BlockNode &X) const { return Index == X.Index; } bool operator!=(const BlockNode &X) const { return Index != X.Index; } @@ -180,11 +199,11 @@ public: bool operator<(const BlockNode &X) const { return Index < X.Index; } bool operator>(const BlockNode &X) const { return Index > X.Index; } - BlockNode() : Index(UINT32_MAX) {} - BlockNode(IndexType Index) : Index(Index) {} - bool isValid() const { return Index <= getMaxIndex(); } - static size_t getMaxIndex() { return UINT32_MAX - 1; } + + static size_t getMaxIndex() { + return std::numeric_limits<uint32_t>::max() - 1; + } }; /// \brief Stats about a block itself. @@ -198,12 +217,13 @@ public: /// Contains the data necessary to represent a loop as a pseudo-node once it's /// packaged. struct LoopData { - typedef SmallVector<std::pair<BlockNode, BlockMass>, 4> ExitMap; - typedef SmallVector<BlockNode, 4> NodeList; - typedef SmallVector<BlockMass, 1> HeaderMassList; + using ExitMap = SmallVector<std::pair<BlockNode, BlockMass>, 4>; + using NodeList = SmallVector<BlockNode, 4>; + using HeaderMassList = SmallVector<BlockMass, 1>; + LoopData *Parent; ///< The parent loop. - bool IsPackaged; ///< Whether this has been packaged. - uint32_t NumHeaders; ///< Number of headers. + bool IsPackaged = false; ///< Whether this has been packaged. + uint32_t NumHeaders = 1; ///< Number of headers. ExitMap Exits; ///< Successor edges (and weights). NodeList Nodes; ///< Header and the members of the loop. HeaderMassList BackedgeMass; ///< Mass returned to each loop header. @@ -211,22 +231,24 @@ public: Scaled64 Scale; LoopData(LoopData *Parent, const BlockNode &Header) - : Parent(Parent), IsPackaged(false), NumHeaders(1), Nodes(1, Header), - BackedgeMass(1) {} + : Parent(Parent), Nodes(1, Header), BackedgeMass(1) {} + template <class It1, class It2> LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther, It2 LastOther) - : Parent(Parent), IsPackaged(false), Nodes(FirstHeader, LastHeader) { + : Parent(Parent), Nodes(FirstHeader, LastHeader) { NumHeaders = Nodes.size(); Nodes.insert(Nodes.end(), FirstOther, LastOther); BackedgeMass.resize(NumHeaders); } + bool isHeader(const BlockNode &Node) const { if (isIrreducible()) return std::binary_search(Nodes.begin(), Nodes.begin() + NumHeaders, Node); return Node == Nodes[0]; } + BlockNode getHeader() const { return Nodes[0]; } bool isIrreducible() const { return NumHeaders > 1; } @@ -241,6 +263,7 @@ public: NodeList::const_iterator members_begin() const { return Nodes.begin() + NumHeaders; } + NodeList::const_iterator members_end() const { return Nodes.end(); } iterator_range<NodeList::const_iterator> members() const { return make_range(members_begin(), members_end()); @@ -249,13 +272,14 @@ public: /// \brief Index of loop information. struct WorkingData { - BlockNode Node; ///< This node. - LoopData *Loop; ///< The loop this block is inside. - BlockMass Mass; ///< Mass distribution from the entry block. + BlockNode Node; ///< This node. + LoopData *Loop = nullptr; ///< The loop this block is inside. + BlockMass Mass; ///< Mass distribution from the entry block. - WorkingData(const BlockNode &Node) : Node(Node), Loop(nullptr) {} + WorkingData(const BlockNode &Node) : Node(Node) {} bool isLoopHeader() const { return Loop && Loop->isHeader(Node); } + bool isDoubleLoopHeader() const { return isLoopHeader() && Loop->Parent && Loop->Parent->isIrreducible() && Loop->Parent->isHeader(Node); @@ -286,6 +310,7 @@ public: auto L = getPackagedLoop(); return L ? L->getHeader() : Node; } + LoopData *getPackagedLoop() const { if (!Loop || !Loop->IsPackaged) return nullptr; @@ -310,8 +335,10 @@ public: /// \brief Has ContainingLoop been packaged up? bool isPackaged() const { return getResolvedNode() != Node; } + /// \brief Has Loop been packaged up? bool isAPackage() const { return isLoopHeader() && Loop->IsPackaged; } + /// \brief Has Loop been packaged up twice? bool isADoublePackage() const { return isDoubleLoopHeader() && Loop->Parent->IsPackaged; @@ -333,10 +360,11 @@ public: /// backedge to the loop header? struct Weight { enum DistType { Local, Exit, Backedge }; - DistType Type; + DistType Type = Local; BlockNode TargetNode; - uint64_t Amount; - Weight() : Type(Local), Amount(0) {} + uint64_t Amount = 0; + + Weight() = default; Weight(DistType Type, BlockNode TargetNode, uint64_t Amount) : Type(Type), TargetNode(TargetNode), Amount(Amount) {} }; @@ -350,18 +378,22 @@ public: /// \a DidOverflow indicates whether \a Total did overflow while adding to /// the distribution. It should never overflow twice. struct Distribution { - typedef SmallVector<Weight, 4> WeightList; - WeightList Weights; ///< Individual successor weights. - uint64_t Total; ///< Sum of all weights. - bool DidOverflow; ///< Whether \a Total did overflow. + using WeightList = SmallVector<Weight, 4>; + + WeightList Weights; ///< Individual successor weights. + uint64_t Total = 0; ///< Sum of all weights. + bool DidOverflow = false; ///< Whether \a Total did overflow. + + Distribution() = default; - Distribution() : Total(0), DidOverflow(false) {} void addLocal(const BlockNode &Node, uint64_t Amount) { add(Node, Amount, Weight::Local); } + void addExit(const BlockNode &Node, uint64_t Amount) { add(Node, Amount, Weight::Exit); } + void addBackedge(const BlockNode &Node, uint64_t Amount) { add(Node, Amount, Weight::Backedge); } @@ -384,12 +416,22 @@ public: /// \brief Data about each block. This is used downstream. std::vector<FrequencyData> Freqs; + /// \brief Whether each block is an irreducible loop header. + /// This is used downstream. + SparseBitVector<> IsIrrLoopHeader; + /// \brief Loop data: see initializeLoops(). std::vector<WorkingData> Working; /// \brief Indexed information about loops. std::list<LoopData> Loops; + /// \brief Virtual destructor. + /// + /// Need a virtual destructor to mask the compiler warning about + /// getBlockName(). + virtual ~BlockFrequencyInfoImplBase() = default; + /// \brief Add all edges out of a packaged loop to the distribution. /// /// Adds all edges from LocalLoopHead to Dist. Calls addToDist() to add each @@ -456,6 +498,8 @@ public: /// the backedges going into each of the loop headers. void adjustLoopHeaderMass(LoopData &Loop); + void distributeIrrLoopHeaderMass(Distribution &Dist); + /// \brief Package up a loop. void packageLoop(LoopData &Loop); @@ -484,6 +528,7 @@ public: const BlockNode &Node) const; Optional<uint64_t> getProfileCountFromFreq(const Function &F, uint64_t Freq) const; + bool isIrrLoopHeader(const BlockNode &Node); void setBlockFreq(const BlockNode &Node, uint64_t Freq); @@ -495,28 +540,24 @@ public: assert(!Freqs.empty()); return Freqs[0].Integer; } - /// \brief Virtual destructor. - /// - /// Need a virtual destructor to mask the compiler warning about - /// getBlockName(). - virtual ~BlockFrequencyInfoImplBase() {} }; namespace bfi_detail { + template <class BlockT> struct TypeMap {}; template <> struct TypeMap<BasicBlock> { - typedef BasicBlock BlockT; - typedef Function FunctionT; - typedef BranchProbabilityInfo BranchProbabilityInfoT; - typedef Loop LoopT; - typedef LoopInfo LoopInfoT; + using BlockT = BasicBlock; + using FunctionT = Function; + using BranchProbabilityInfoT = BranchProbabilityInfo; + using LoopT = Loop; + using LoopInfoT = LoopInfo; }; template <> struct TypeMap<MachineBasicBlock> { - typedef MachineBasicBlock BlockT; - typedef MachineFunction FunctionT; - typedef MachineBranchProbabilityInfo BranchProbabilityInfoT; - typedef MachineLoop LoopT; - typedef MachineLoopInfo LoopInfoT; + using BlockT = MachineBasicBlock; + using FunctionT = MachineFunction; + using BranchProbabilityInfoT = MachineBranchProbabilityInfo; + using LoopT = MachineLoop; + using LoopInfoT = MachineLoopInfo; }; /// \brief Get the name of a MachineBasicBlock. @@ -554,25 +595,27 @@ template <> inline std::string getBlockName(const BasicBlock *BB) { /// and it explicitly lists predecessors and successors. The initialization /// that relies on \c MachineBasicBlock is defined in the header. struct IrreducibleGraph { - typedef BlockFrequencyInfoImplBase BFIBase; + using BFIBase = BlockFrequencyInfoImplBase; BFIBase &BFI; - typedef BFIBase::BlockNode BlockNode; + using BlockNode = BFIBase::BlockNode; struct IrrNode { BlockNode Node; - unsigned NumIn; + unsigned NumIn = 0; std::deque<const IrrNode *> Edges; - IrrNode(const BlockNode &Node) : Node(Node), NumIn(0) {} - typedef std::deque<const IrrNode *>::const_iterator iterator; + IrrNode(const BlockNode &Node) : Node(Node) {} + + using iterator = std::deque<const IrrNode *>::const_iterator; + iterator pred_begin() const { return Edges.begin(); } iterator succ_begin() const { return Edges.begin() + NumIn; } iterator pred_end() const { return succ_begin(); } iterator succ_end() const { return Edges.end(); } }; BlockNode Start; - const IrrNode *StartIrr; + const IrrNode *StartIrr = nullptr; std::vector<IrrNode> Nodes; SmallDenseMap<uint32_t, IrrNode *, 4> Lookup; @@ -587,8 +630,7 @@ struct IrreducibleGraph { /// user of this. template <class BlockEdgesAdder> IrreducibleGraph(BFIBase &BFI, const BFIBase::LoopData *OuterLoop, - BlockEdgesAdder addBlockEdges) - : BFI(BFI), StartIrr(nullptr) { + BlockEdgesAdder addBlockEdges) : BFI(BFI) { initialize(OuterLoop, addBlockEdges); } @@ -597,10 +639,12 @@ struct IrreducibleGraph { BlockEdgesAdder addBlockEdges); void addNodesInLoop(const BFIBase::LoopData &OuterLoop); void addNodesInFunction(); + void addNode(const BlockNode &Node) { Nodes.emplace_back(Node); BFI.Working[Node.Index].getMass() = BlockMass::getEmpty(); } + void indexNodes(); template <class BlockEdgesAdder> void addEdges(const BlockNode &Node, const BFIBase::LoopData *OuterLoop, @@ -608,6 +652,7 @@ struct IrreducibleGraph { void addEdge(IrrNode &Irr, const BlockNode &Succ, const BFIBase::LoopData *OuterLoop); }; + template <class BlockEdgesAdder> void IrreducibleGraph::initialize(const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges) { @@ -622,6 +667,7 @@ void IrreducibleGraph::initialize(const BFIBase::LoopData *OuterLoop, } StartIrr = Lookup[Start.Index]; } + template <class BlockEdgesAdder> void IrreducibleGraph::addEdges(const BlockNode &Node, const BFIBase::LoopData *OuterLoop, @@ -638,7 +684,8 @@ void IrreducibleGraph::addEdges(const BlockNode &Node, else addBlockEdges(*this, Irr, OuterLoop); } -} + +} // end namespace bfi_detail /// \brief Shared implementation for block frequency analysis. /// @@ -794,28 +841,27 @@ void IrreducibleGraph::addEdges(const BlockNode &Node, /// (Running this until fixed point would "solve" the geometric /// series by simulation.) template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { - typedef typename bfi_detail::TypeMap<BT>::BlockT BlockT; - typedef typename bfi_detail::TypeMap<BT>::FunctionT FunctionT; - typedef typename bfi_detail::TypeMap<BT>::BranchProbabilityInfoT - BranchProbabilityInfoT; - typedef typename bfi_detail::TypeMap<BT>::LoopT LoopT; - typedef typename bfi_detail::TypeMap<BT>::LoopInfoT LoopInfoT; - // This is part of a workaround for a GCC 4.7 crash on lambdas. friend struct bfi_detail::BlockEdgesAdder<BT>; - typedef GraphTraits<const BlockT *> Successor; - typedef GraphTraits<Inverse<const BlockT *>> Predecessor; + using BlockT = typename bfi_detail::TypeMap<BT>::BlockT; + using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT; + using BranchProbabilityInfoT = + typename bfi_detail::TypeMap<BT>::BranchProbabilityInfoT; + using LoopT = typename bfi_detail::TypeMap<BT>::LoopT; + using LoopInfoT = typename bfi_detail::TypeMap<BT>::LoopInfoT; + using Successor = GraphTraits<const BlockT *>; + using Predecessor = GraphTraits<Inverse<const BlockT *>>; - const BranchProbabilityInfoT *BPI; - const LoopInfoT *LI; - const FunctionT *F; + const BranchProbabilityInfoT *BPI = nullptr; + const LoopInfoT *LI = nullptr; + const FunctionT *F = nullptr; // All blocks in reverse postorder. std::vector<const BlockT *> RPOT; DenseMap<const BlockT *, BlockNode> Nodes; - typedef typename std::vector<const BlockT *>::const_iterator rpot_iterator; + using rpot_iterator = typename std::vector<const BlockT *>::const_iterator; rpot_iterator rpot_begin() const { return RPOT.begin(); } rpot_iterator rpot_end() const { return RPOT.end(); } @@ -913,25 +959,35 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { } public: + BlockFrequencyInfoImpl() = default; + const FunctionT *getFunction() const { return F; } void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI, const LoopInfoT &LI); - BlockFrequencyInfoImpl() : BPI(nullptr), LI(nullptr), F(nullptr) {} using BlockFrequencyInfoImplBase::getEntryFreq; + BlockFrequency getBlockFreq(const BlockT *BB) const { return BlockFrequencyInfoImplBase::getBlockFreq(getNode(BB)); } + Optional<uint64_t> getBlockProfileCount(const Function &F, const BlockT *BB) const { return BlockFrequencyInfoImplBase::getBlockProfileCount(F, getNode(BB)); } + Optional<uint64_t> getProfileCountFromFreq(const Function &F, uint64_t Freq) const { return BlockFrequencyInfoImplBase::getProfileCountFromFreq(F, Freq); } + + bool isIrrLoopHeader(const BlockT *BB) { + return BlockFrequencyInfoImplBase::isIrrLoopHeader(getNode(BB)); + } + void setBlockFreq(const BlockT *BB, uint64_t Freq); + Scaled64 getFloatingBlockFreq(const BlockT *BB) const { return BlockFrequencyInfoImplBase::getFloatingBlockFreq(getNode(BB)); } @@ -950,9 +1006,10 @@ public: /// \a BlockFrequencyInfoImplBase::print() only knows reverse post-order, so /// we need to override it here. raw_ostream &print(raw_ostream &OS) const override; - using BlockFrequencyInfoImplBase::dump; + using BlockFrequencyInfoImplBase::dump; using BlockFrequencyInfoImplBase::printBlockFreq; + raw_ostream &printBlockFreq(raw_ostream &OS, const BlockT *BB) const { return BlockFrequencyInfoImplBase::printBlockFreq(OS, getNode(BB)); } @@ -1096,17 +1153,59 @@ bool BlockFrequencyInfoImpl<BT>::computeMassInLoop(LoopData &Loop) { DEBUG(dbgs() << "compute-mass-in-loop: " << getLoopName(Loop) << "\n"); if (Loop.isIrreducible()) { - BlockMass Remaining = BlockMass::getFull(); + DEBUG(dbgs() << "isIrreducible = true\n"); + Distribution Dist; + unsigned NumHeadersWithWeight = 0; + Optional<uint64_t> MinHeaderWeight; + DenseSet<uint32_t> HeadersWithoutWeight; + HeadersWithoutWeight.reserve(Loop.NumHeaders); for (uint32_t H = 0; H < Loop.NumHeaders; ++H) { - auto &Mass = Working[Loop.Nodes[H].Index].getMass(); - Mass = Remaining * BranchProbability(1, Loop.NumHeaders - H); - Remaining -= Mass; + auto &HeaderNode = Loop.Nodes[H]; + const BlockT *Block = getBlock(HeaderNode); + IsIrrLoopHeader.set(Loop.Nodes[H].Index); + Optional<uint64_t> HeaderWeight = Block->getIrrLoopHeaderWeight(); + if (!HeaderWeight) { + DEBUG(dbgs() << "Missing irr loop header metadata on " + << getBlockName(HeaderNode) << "\n"); + HeadersWithoutWeight.insert(H); + continue; + } + DEBUG(dbgs() << getBlockName(HeaderNode) + << " has irr loop header weight " << HeaderWeight.getValue() + << "\n"); + NumHeadersWithWeight++; + uint64_t HeaderWeightValue = HeaderWeight.getValue(); + if (!MinHeaderWeight || HeaderWeightValue < MinHeaderWeight) + MinHeaderWeight = HeaderWeightValue; + if (HeaderWeightValue) { + Dist.addLocal(HeaderNode, HeaderWeightValue); + } + } + // As a heuristic, if some headers don't have a weight, give them the + // minimium weight seen (not to disrupt the existing trends too much by + // using a weight that's in the general range of the other headers' weights, + // and the minimum seems to perform better than the average.) + // FIXME: better update in the passes that drop the header weight. + // If no headers have a weight, give them even weight (use weight 1). + if (!MinHeaderWeight) + MinHeaderWeight = 1; + for (uint32_t H : HeadersWithoutWeight) { + auto &HeaderNode = Loop.Nodes[H]; + assert(!getBlock(HeaderNode)->getIrrLoopHeaderWeight() && + "Shouldn't have a weight metadata"); + uint64_t MinWeight = MinHeaderWeight.getValue(); + DEBUG(dbgs() << "Giving weight " << MinWeight + << " to " << getBlockName(HeaderNode) << "\n"); + if (MinWeight) + Dist.addLocal(HeaderNode, MinWeight); } + distributeIrrLoopHeaderMass(Dist); for (const BlockNode &M : Loop.Nodes) if (!propagateMassToSuccessors(&Loop, M)) llvm_unreachable("unhandled irreducible control flow"); - - adjustLoopHeaderMass(Loop); + if (NumHeadersWithWeight == 0) + // No headers have a metadata. Adjust header mass. + adjustLoopHeaderMass(Loop); } else { Working[Loop.getHeader().Index].getMass() = BlockMass::getFull(); if (!propagateMassToSuccessors(&Loop, Loop.getHeader())) @@ -1153,14 +1252,17 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::computeMassInFunction() { /// \note This should be a lambda, but that crashes GCC 4.7. namespace bfi_detail { + template <class BT> struct BlockEdgesAdder { - typedef BT BlockT; - typedef BlockFrequencyInfoImplBase::LoopData LoopData; - typedef GraphTraits<const BlockT *> Successor; + using BlockT = BT; + using LoopData = BlockFrequencyInfoImplBase::LoopData; + using Successor = GraphTraits<const BlockT *>; const BlockFrequencyInfoImpl<BT> &BFI; + explicit BlockEdgesAdder(const BlockFrequencyInfoImpl<BT> &BFI) : BFI(BFI) {} + void operator()(IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr, const LoopData *OuterLoop) { const BlockT *BB = BFI.RPOT[Irr.Node.Index]; @@ -1168,7 +1270,9 @@ template <class BT> struct BlockEdgesAdder { G.addEdge(Irr, BFI.getNode(Succ), OuterLoop); } }; -} + +} // end namespace bfi_detail + template <class BT> void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass( LoopData *OuterLoop, std::list<LoopData>::iterator Insert) { @@ -1177,6 +1281,7 @@ void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass( else dbgs() << "function\n"); using namespace bfi_detail; + // Ideally, addBlockEdges() would be declared here as a lambda, but that // crashes GCC 4.7. BlockEdgesAdder<BT> addBlockEdges(*this); @@ -1209,9 +1314,12 @@ BlockFrequencyInfoImpl<BT>::propagateMassToSuccessors(LoopData *OuterLoop, return false; } else { const BlockT *BB = getBlock(Node); - for (const auto Succ : children<const BlockT *>(BB)) - if (!addToDist(Dist, OuterLoop, Node, getNode(Succ), - getWeightFromBranchProb(BPI->getEdgeProbability(BB, Succ)))) + for (auto SI = GraphTraits<const BlockT *>::child_begin(BB), + SE = GraphTraits<const BlockT *>::child_end(BB); + SI != SE; ++SI) + if (!addToDist( + Dist, OuterLoop, Node, getNode(*SI), + getWeightFromBranchProb(BPI->getEdgeProbability(BB, SI)))) // Irreducible backedge. return false; } @@ -1230,7 +1338,15 @@ raw_ostream &BlockFrequencyInfoImpl<BT>::print(raw_ostream &OS) const { for (const BlockT &BB : *F) { OS << " - " << bfi_detail::getBlockName(&BB) << ": float = "; getFloatingBlockFreq(&BB).print(OS, 5) - << ", int = " << getBlockFreq(&BB).getFrequency() << "\n"; + << ", int = " << getBlockFreq(&BB).getFrequency(); + if (Optional<uint64_t> ProfileCount = + BlockFrequencyInfoImplBase::getBlockProfileCount( + F->getFunction(), getNode(&BB))) + OS << ", count = " << ProfileCount.getValue(); + if (Optional<uint64_t> IrrLoopHeaderWeight = + BB.getIrrLoopHeaderWeight()) + OS << ", irr_loop_header_weight = " << IrrLoopHeaderWeight.getValue(); + OS << "\n"; } // Add an extra newline for readability. @@ -1245,15 +1361,16 @@ enum GVDAGType { GVDT_None, GVDT_Fraction, GVDT_Integer, GVDT_Count }; template <class BlockFrequencyInfoT, class BranchProbabilityInfoT> struct BFIDOTGraphTraitsBase : public DefaultDOTGraphTraits { + using GTraits = GraphTraits<BlockFrequencyInfoT *>; + using NodeRef = typename GTraits::NodeRef; + using EdgeIter = typename GTraits::ChildIteratorType; + using NodeIter = typename GTraits::nodes_iterator; + + uint64_t MaxFrequency = 0; + explicit BFIDOTGraphTraitsBase(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} - typedef GraphTraits<BlockFrequencyInfoT *> GTraits; - typedef typename GTraits::NodeRef NodeRef; - typedef typename GTraits::ChildIteratorType EdgeIter; - typedef typename GTraits::nodes_iterator NodeIter; - - uint64_t MaxFrequency = 0; static std::string getGraphName(const BlockFrequencyInfoT *G) { return G->getFunction()->getName(); } diff --git a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index 94d3d4de6c9d..417b64978811 100644 --- a/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/contrib/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -1,4 +1,4 @@ -//===--- BranchProbabilityInfo.h - Branch Probability Analysis --*- C++ -*-===// +//===- BranchProbabilityInfo.h - Branch Probability Analysis ----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,19 +15,28 @@ #define LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/BranchProbability.h" +#include "llvm/Support/Casting.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <utility> namespace llvm { + +class Function; class LoopInfo; -class TargetLibraryInfo; class raw_ostream; +class TargetLibraryInfo; +class Value; /// \brief Analysis providing branch probability information. /// @@ -43,7 +52,8 @@ class raw_ostream; /// value 10. class BranchProbabilityInfo { public: - BranchProbabilityInfo() {} + BranchProbabilityInfo() = default; + BranchProbabilityInfo(const Function &F, const LoopInfo &LI, const TargetLibraryInfo *TLI = nullptr) { calculate(F, LI, TLI); @@ -54,6 +64,9 @@ public: PostDominatedByUnreachable(std::move(Arg.PostDominatedByUnreachable)), PostDominatedByColdCall(std::move(Arg.PostDominatedByColdCall)) {} + BranchProbabilityInfo(const BranchProbabilityInfo &) = delete; + BranchProbabilityInfo &operator=(const BranchProbabilityInfo &) = delete; + BranchProbabilityInfo &operator=(BranchProbabilityInfo &&RHS) { releaseMemory(); Probs = std::move(RHS.Probs); @@ -124,14 +137,21 @@ public: /// Forget analysis results for the given basic block. void eraseBlock(const BasicBlock *BB); -private: - void operator=(const BranchProbabilityInfo &) = delete; - BranchProbabilityInfo(const BranchProbabilityInfo &) = delete; + // Use to track SCCs for handling irreducible loops. + using SccMap = DenseMap<const BasicBlock *, int>; + using SccHeaderMap = DenseMap<const BasicBlock *, bool>; + using SccHeaderMaps = std::vector<SccHeaderMap>; + struct SccInfo { + SccMap SccNums; + SccHeaderMaps SccHeaders; + }; +private: // We need to store CallbackVH's in order to correctly handle basic block // removal. class BasicBlockCallbackVH final : public CallbackVH { BranchProbabilityInfo *BPI; + void deleted() override { assert(BPI != nullptr); BPI->eraseBlock(cast<BasicBlock>(getValPtr())); @@ -139,14 +159,15 @@ private: } public: - BasicBlockCallbackVH(const Value *V, BranchProbabilityInfo *BPI=nullptr) + BasicBlockCallbackVH(const Value *V, BranchProbabilityInfo *BPI = nullptr) : CallbackVH(const_cast<Value *>(V)), BPI(BPI) {} }; + DenseSet<BasicBlockCallbackVH, DenseMapInfo<Value*>> Handles; // Since we allow duplicate edges from one basic block to another, we use // a pair (PredBlock and an index in the successors) to specify an edge. - typedef std::pair<const BasicBlock *, unsigned> Edge; + using Edge = std::pair<const BasicBlock *, unsigned>; // Default weight value. Used when we don't have information about the edge. // TODO: DEFAULT_WEIGHT makes sense during static predication, when none of @@ -173,7 +194,8 @@ private: bool calcMetadataWeights(const BasicBlock *BB); bool calcColdCallHeuristics(const BasicBlock *BB); bool calcPointerHeuristics(const BasicBlock *BB); - bool calcLoopBranchHeuristics(const BasicBlock *BB, const LoopInfo &LI); + bool calcLoopBranchHeuristics(const BasicBlock *BB, const LoopInfo &LI, + SccInfo &SccI); bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI); bool calcFloatingPointHeuristics(const BasicBlock *BB); bool calcInvokeHeuristics(const BasicBlock *BB); @@ -183,11 +205,12 @@ private: class BranchProbabilityAnalysis : public AnalysisInfoMixin<BranchProbabilityAnalysis> { friend AnalysisInfoMixin<BranchProbabilityAnalysis>; + static AnalysisKey Key; public: - /// \brief Provide the result typedef for this analysis pass. - typedef BranchProbabilityInfo Result; + /// \brief Provide the result type for this analysis pass. + using Result = BranchProbabilityInfo; /// \brief Run the analysis pass over a function and produce BPI. BranchProbabilityInfo run(Function &F, FunctionAnalysisManager &AM); @@ -200,6 +223,7 @@ class BranchProbabilityPrinterPass public: explicit BranchProbabilityPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; @@ -224,6 +248,6 @@ public: void print(raw_ostream &OS, const Module *M = nullptr) const override; }; -} +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H diff --git a/contrib/llvm/include/llvm/Analysis/CFG.h b/contrib/llvm/include/llvm/Analysis/CFG.h index 35165f4061f1..eab64176f0d7 100644 --- a/contrib/llvm/include/llvm/Analysis/CFG.h +++ b/contrib/llvm/include/llvm/Analysis/CFG.h @@ -59,7 +59,7 @@ bool isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum, /// This function is linear with respect to the number of blocks in the CFG, /// walking down successors from From to reach To, with a fixed threshold. /// Using DT or LI allows us to answer more quickly. LI reduces the cost of -/// an entire loop of any number of blocsk to be the same as the cost of a +/// an entire loop of any number of blocks to be the same as the cost of a /// single block. DT reduces the cost by allowing the search to terminate when /// we find a block that dominates the block containing 'To'. DT is most useful /// on branchy code but not loops, and LI is most useful on code with loops but diff --git a/contrib/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h index 4146ad4d18ac..6239d5309581 100644 --- a/contrib/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/CFLAndersAliasAnalysis.h @@ -1,4 +1,4 @@ -//=- CFLAndersAliasAnalysis.h - Unification-based Alias Analysis ---*- C++-*-=// +//==- CFLAndersAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==// // // The LLVM Compiler Infrastructure // @@ -19,25 +19,31 @@ #include "llvm/ADT/Optional.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFLAliasAnalysisUtils.h" -#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include <forward_list> +#include <memory> namespace llvm { +class Function; +class MemoryLocation; class TargetLibraryInfo; namespace cflaa { + struct AliasSummary; -} + +} // end namespace cflaa class CFLAndersAAResult : public AAResultBase<CFLAndersAAResult> { friend AAResultBase<CFLAndersAAResult>; + class FunctionInfo; public: - explicit CFLAndersAAResult(const TargetLibraryInfo &); - CFLAndersAAResult(CFLAndersAAResult &&); + explicit CFLAndersAAResult(const TargetLibraryInfo &TLI); + CFLAndersAAResult(CFLAndersAAResult &&RHS); ~CFLAndersAAResult(); /// Handle invalidation events from the new pass manager. @@ -46,6 +52,7 @@ public: FunctionAnalysisManager::Invalidator &) { return false; } + /// Evict the given function from cache void evict(const Function *Fn); @@ -85,10 +92,11 @@ private: /// in particular to leverage invalidation to trigger re-computation. class CFLAndersAA : public AnalysisInfoMixin<CFLAndersAA> { friend AnalysisInfoMixin<CFLAndersAA>; + static AnalysisKey Key; public: - typedef CFLAndersAAResult Result; + using Result = CFLAndersAAResult; CFLAndersAAResult run(Function &F, FunctionAnalysisManager &AM); }; @@ -109,12 +117,10 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override; }; -//===--------------------------------------------------------------------===// -// // createCFLAndersAAWrapperPass - This pass implements a set-based approach to // alias analysis. -// ImmutablePass *createCFLAndersAAWrapperPass(); -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_CFLANDERSALIASANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h index fd3fa5febcdf..ee9e29046af8 100644 --- a/contrib/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/CFLSteensAliasAnalysis.h @@ -1,4 +1,4 @@ -//=- CFLSteensAliasAnalysis.h - Unification-based Alias Analysis ---*- C++-*-=// +//==- CFLSteensAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==// // // The LLVM Compiler Infrastructure // @@ -16,30 +16,34 @@ #define LLVM_ANALYSIS_CFLSTEENSALIASANALYSIS_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFLAliasAnalysisUtils.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/ValueHandle.h" +#include "llvm/Analysis/MemoryLocation.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include <forward_list> +#include <memory> namespace llvm { +class Function; class TargetLibraryInfo; namespace cflaa { + struct AliasSummary; -} + +} // end namespace cflaa class CFLSteensAAResult : public AAResultBase<CFLSteensAAResult> { friend AAResultBase<CFLSteensAAResult>; + class FunctionInfo; public: - explicit CFLSteensAAResult(const TargetLibraryInfo &); + explicit CFLSteensAAResult(const TargetLibraryInfo &TLI); CFLSteensAAResult(CFLSteensAAResult &&Arg); ~CFLSteensAAResult(); @@ -68,7 +72,7 @@ public: AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { if (LocA.Ptr == LocB.Ptr) - return LocA.Size == LocB.Size ? MustAlias : PartialAlias; + return MustAlias; // Comparisons between global variables and other constants should be // handled by BasicAA. @@ -105,10 +109,11 @@ private: /// in particular to leverage invalidation to trigger re-computation of sets. class CFLSteensAA : public AnalysisInfoMixin<CFLSteensAA> { friend AnalysisInfoMixin<CFLSteensAA>; + static AnalysisKey Key; public: - typedef CFLSteensAAResult Result; + using Result = CFLSteensAAResult; CFLSteensAAResult run(Function &F, FunctionAnalysisManager &AM); }; @@ -129,12 +134,10 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override; }; -//===--------------------------------------------------------------------===// -// // createCFLSteensAAWrapperPass - This pass implements a set-based approach to // alias analysis. -// ImmutablePass *createCFLSteensAAWrapperPass(); -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_CFLSTEENSALIASANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h index 32868cbecdcf..8123cbad22ff 100644 --- a/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/contrib/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -89,29 +89,44 @@ #ifndef LLVM_ANALYSIS_CGSCCPASSMANAGER_H #define LLVM_ANALYSIS_CGSCCPASSMANAGER_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/PriorityWorklist.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <utility> namespace llvm { struct CGSCCUpdateResult; +class Module; + +// Allow debug logging in this inline function. +#define DEBUG_TYPE "cgscc" /// Extern template declaration for the analysis set for this IR unit. extern template class AllAnalysesOn<LazyCallGraph::SCC>; extern template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>; + /// \brief The CGSCC analysis manager. /// /// See the documentation for the AnalysisManager template for detail -/// documentation. This typedef serves as a convenient way to refer to this +/// documentation. This type serves as a convenient way to refer to this /// construct in the adaptors and proxies used to integrate this into the larger /// pass manager infrastructure. -typedef AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &> - CGSCCAnalysisManager; +using CGSCCAnalysisManager = + AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>; // Explicit specialization and instantiation declarations for the pass manager. // See the comments on the definition of the specialization for details on how @@ -129,10 +144,10 @@ extern template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, /// /// See the documentation for the PassManager template for details. It runs /// a sequence of SCC passes over each SCC that the manager is run over. This -/// typedef serves as a convenient way to refer to this construct. -typedef PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, - CGSCCUpdateResult &> - CGSCCPassManager; +/// type serves as a convenient way to refer to this construct. +using CGSCCPassManager = + PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, + CGSCCUpdateResult &>; /// An explicit specialization of the require analysis template pass. template <typename AnalysisT> @@ -149,8 +164,8 @@ struct RequireAnalysisPass<AnalysisT, LazyCallGraph::SCC, CGSCCAnalysisManager, }; /// A proxy from a \c CGSCCAnalysisManager to a \c Module. -typedef InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module> - CGSCCAnalysisManagerModuleProxy; +using CGSCCAnalysisManagerModuleProxy = + InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>; /// We need a specialized result for the \c CGSCCAnalysisManagerModuleProxy so /// it can have access to the call graph in order to walk all the SCCs when @@ -193,10 +208,11 @@ extern template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>; extern template class OuterAnalysisManagerProxy< ModuleAnalysisManager, LazyCallGraph::SCC, LazyCallGraph &>; + /// A proxy from a \c ModuleAnalysisManager to an \c SCC. -typedef OuterAnalysisManagerProxy<ModuleAnalysisManager, LazyCallGraph::SCC, - LazyCallGraph &> - ModuleAnalysisManagerCGSCCProxy; +using ModuleAnalysisManagerCGSCCProxy = + OuterAnalysisManagerProxy<ModuleAnalysisManager, LazyCallGraph::SCC, + LazyCallGraph &>; /// Support structure for SCC passes to communicate updates the call graph back /// to the CGSCC pass manager infrsatructure. @@ -275,6 +291,15 @@ struct CGSCCUpdateResult { /// non-null and can be used to continue processing the "top" of the /// post-order walk. LazyCallGraph::SCC *UpdatedC; + + /// A hacky area where the inliner can retain history about inlining + /// decisions that mutated the call graph's SCC structure in order to avoid + /// infinite inlining. See the comments in the inliner's CG update logic. + /// + /// FIXME: Keeping this here seems like a big layering issue, we should look + /// for a better technique. + SmallDenseSet<std::pair<LazyCallGraph::Node *, LazyCallGraph::SCC *>, 4> + &InlinedInternalEdges; }; /// \brief The core module pass which does a post-order walk of the SCCs and @@ -290,21 +315,23 @@ template <typename CGSCCPassT> class ModuleToPostOrderCGSCCPassAdaptor : public PassInfoMixin<ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>> { public: - explicit ModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass, bool DebugLogging = false) - : Pass(std::move(Pass)), DebugLogging(DebugLogging) {} + explicit ModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) + : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC // refuses to generate them. ModuleToPostOrderCGSCCPassAdaptor( const ModuleToPostOrderCGSCCPassAdaptor &Arg) - : Pass(Arg.Pass), DebugLogging(Arg.DebugLogging) {} + : Pass(Arg.Pass) {} + ModuleToPostOrderCGSCCPassAdaptor(ModuleToPostOrderCGSCCPassAdaptor &&Arg) - : Pass(std::move(Arg.Pass)), DebugLogging(Arg.DebugLogging) {} + : Pass(std::move(Arg.Pass)) {} + friend void swap(ModuleToPostOrderCGSCCPassAdaptor &LHS, ModuleToPostOrderCGSCCPassAdaptor &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - swap(LHS.DebugLogging, RHS.DebugLogging); + std::swap(LHS.Pass, RHS.Pass); } + ModuleToPostOrderCGSCCPassAdaptor & operator=(ModuleToPostOrderCGSCCPassAdaptor RHS) { swap(*this, RHS); @@ -330,8 +357,12 @@ public: SmallPtrSet<LazyCallGraph::RefSCC *, 4> InvalidRefSCCSet; SmallPtrSet<LazyCallGraph::SCC *, 4> InvalidSCCSet; - CGSCCUpdateResult UR = {RCWorklist, CWorklist, InvalidRefSCCSet, - InvalidSCCSet, nullptr, nullptr}; + SmallDenseSet<std::pair<LazyCallGraph::Node *, LazyCallGraph::SCC *>, 4> + InlinedInternalEdges; + + CGSCCUpdateResult UR = {RCWorklist, CWorklist, InvalidRefSCCSet, + InvalidSCCSet, nullptr, nullptr, + InlinedInternalEdges}; PreservedAnalyses PA = PreservedAnalyses::all(); CG.buildRefSCCs(); @@ -356,20 +387,19 @@ public: do { LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); if (InvalidRefSCCSet.count(RC)) { - if (DebugLogging) - dbgs() << "Skipping an invalid RefSCC...\n"; + DEBUG(dbgs() << "Skipping an invalid RefSCC...\n"); continue; } assert(CWorklist.empty() && "Should always start with an empty SCC worklist"); - if (DebugLogging) - dbgs() << "Running an SCC pass across the RefSCC: " << *RC << "\n"; + DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC + << "\n"); // Push the initial SCCs in reverse post-order as we'll pop off the the // back and so see this in post-order. - for (LazyCallGraph::SCC &C : reverse(*RC)) + for (LazyCallGraph::SCC &C : llvm::reverse(*RC)) CWorklist.insert(&C); do { @@ -379,14 +409,12 @@ public: // other RefSCCs should be queued above, so we just need to skip both // scenarios here. if (InvalidSCCSet.count(C)) { - if (DebugLogging) - dbgs() << "Skipping an invalid SCC...\n"; + DEBUG(dbgs() << "Skipping an invalid SCC...\n"); continue; } if (&C->getOuterRefSCC() != RC) { - if (DebugLogging) - dbgs() << "Skipping an SCC that is now part of some other " - "RefSCC...\n"; + DEBUG(dbgs() << "Skipping an SCC that is now part of some other " + "RefSCC...\n"); continue; } @@ -401,13 +429,26 @@ public: UR.UpdatedC = nullptr; PreservedAnalyses PassPA = Pass.run(*C, CGAM, CG, UR); + // Update the SCC and RefSCC if necessary. + C = UR.UpdatedC ? UR.UpdatedC : C; + RC = UR.UpdatedRC ? UR.UpdatedRC : RC; + + // If the CGSCC pass wasn't able to provide a valid updated SCC, + // the current SCC may simply need to be skipped if invalid. + if (UR.InvalidatedSCCs.count(C)) { + DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); + break; + } + // Check that we didn't miss any update scenario. + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + // We handle invalidating the CGSCC analysis manager's information // for the (potentially updated) SCC here. Note that any other SCCs // whose structure has changed should have been invalidated by // whatever was updating the call graph. This SCC gets invalidated // late as it contains the nodes that were actively being // processed. - CGAM.invalidate(*(UR.UpdatedC ? UR.UpdatedC : C), PassPA); + CGAM.invalidate(*C, PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -422,19 +463,21 @@ public: // apart, at most converging on a DAG of single nodes. // FIXME: If we ever start having RefSCC passes, we'll want to // iterate there too. - RC = UR.UpdatedRC ? UR.UpdatedRC : RC; - C = UR.UpdatedC ? UR.UpdatedC : C; - if (DebugLogging && UR.UpdatedC) - dbgs() << "Re-running SCC passes after a refinement of the " - "current SCC: " - << *UR.UpdatedC << "\n"; + if (UR.UpdatedC) + DEBUG(dbgs() << "Re-running SCC passes after a refinement of the " + "current SCC: " + << *UR.UpdatedC << "\n"); // Note that both `C` and `RC` may at this point refer to deleted, // invalid SCC and RefSCCs respectively. But we will short circuit // the processing when we check them in the loop above. } while (UR.UpdatedC); - } while (!CWorklist.empty()); + + // We only need to keep internal inlined edge information within + // a RefSCC, clear it to save on space and let the next time we visit + // any of these functions have a fresh start. + InlinedInternalEdges.clear(); } while (!RCWorklist.empty()); } @@ -449,15 +492,14 @@ public: private: CGSCCPassT Pass; - bool DebugLogging; }; /// \brief A function to deduce a function pass type and wrap it in the /// templated adaptor. template <typename CGSCCPassT> ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT> -createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass, bool DebugLogging = false) { - return ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>(std::move(Pass), DebugLogging); +createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) { + return ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>(std::move(Pass)); } /// A proxy from a \c FunctionAnalysisManager to an \c SCC. @@ -490,13 +532,15 @@ public: private: friend AnalysisInfoMixin<FunctionAnalysisManagerCGSCCProxy>; + static AnalysisKey Key; }; extern template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>; + /// A proxy from a \c CGSCCAnalysisManager to a \c Function. -typedef OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function> - CGSCCAnalysisManagerFunctionProxy; +using CGSCCAnalysisManagerFunctionProxy = + OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>; /// Helper to update the call graph after running a function pass. /// @@ -506,7 +550,7 @@ typedef OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function> /// update result struct for the overall CGSCC walk. LazyCallGraph::SCC &updateCGAndAnalysisManagerForFunctionPass( LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, - CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, bool DebugLogging = false); + CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR); /// \brief Adaptor that maps from a SCC to its functions. /// @@ -520,20 +564,22 @@ template <typename FunctionPassT> class CGSCCToFunctionPassAdaptor : public PassInfoMixin<CGSCCToFunctionPassAdaptor<FunctionPassT>> { public: - explicit CGSCCToFunctionPassAdaptor(FunctionPassT Pass, bool DebugLogging = false) - : Pass(std::move(Pass)), DebugLogging(DebugLogging) {} + explicit CGSCCToFunctionPassAdaptor(FunctionPassT Pass) + : Pass(std::move(Pass)) {} + // We have to explicitly define all the special member functions because MSVC // refuses to generate them. CGSCCToFunctionPassAdaptor(const CGSCCToFunctionPassAdaptor &Arg) - : Pass(Arg.Pass), DebugLogging(Arg.DebugLogging) {} + : Pass(Arg.Pass) {} + CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg) - : Pass(std::move(Arg.Pass)), DebugLogging(Arg.DebugLogging) {} + : Pass(std::move(Arg.Pass)) {} + friend void swap(CGSCCToFunctionPassAdaptor &LHS, CGSCCToFunctionPassAdaptor &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - swap(LHS.DebugLogging, RHS.DebugLogging); + std::swap(LHS.Pass, RHS.Pass); } + CGSCCToFunctionPassAdaptor &operator=(CGSCCToFunctionPassAdaptor RHS) { swap(*this, RHS); return *this; @@ -555,8 +601,7 @@ public: // a pointer we can overwrite. LazyCallGraph::SCC *CurrentC = &C; - if (DebugLogging) - dbgs() << "Running function passes across an SCC: " << C << "\n"; + DEBUG(dbgs() << "Running function passes across an SCC: " << C << "\n"); PreservedAnalyses PA = PreservedAnalyses::all(); for (LazyCallGraph::Node *N : Nodes) { @@ -582,8 +627,8 @@ public: // a smaller, more refined SCC. auto PAC = PA.getChecker<LazyCallGraphAnalysis>(); if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) { - CurrentC = &updateCGAndAnalysisManagerForFunctionPass( - CG, *CurrentC, *N, AM, UR, DebugLogging); + CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N, + AM, UR); assert( CG.lookupSCC(*N) == CurrentC && "Current SCC not updated to the SCC containing the current node!"); @@ -605,16 +650,14 @@ public: private: FunctionPassT Pass; - bool DebugLogging; }; /// \brief A function to deduce a function pass type and wrap it in the /// templated adaptor. template <typename FunctionPassT> CGSCCToFunctionPassAdaptor<FunctionPassT> -createCGSCCToFunctionPassAdaptor(FunctionPassT Pass, bool DebugLogging = false) { - return CGSCCToFunctionPassAdaptor<FunctionPassT>(std::move(Pass), - DebugLogging); +createCGSCCToFunctionPassAdaptor(FunctionPassT Pass) { + return CGSCCToFunctionPassAdaptor<FunctionPassT>(std::move(Pass)); } /// A helper that repeats an SCC pass each time an indirect call is refined to @@ -635,10 +678,8 @@ template <typename PassT> class DevirtSCCRepeatedPass : public PassInfoMixin<DevirtSCCRepeatedPass<PassT>> { public: - explicit DevirtSCCRepeatedPass(PassT Pass, int MaxIterations, - bool DebugLogging = false) - : Pass(std::move(Pass)), MaxIterations(MaxIterations), - DebugLogging(DebugLogging) {} + explicit DevirtSCCRepeatedPass(PassT Pass, int MaxIterations) + : Pass(std::move(Pass)), MaxIterations(MaxIterations) {} /// Runs the wrapped pass up to \c MaxIterations on the SCC, iterating /// whenever an indirect call is refined. @@ -716,16 +757,15 @@ public: if (!F) return false; - if (DebugLogging) - dbgs() << "Found devirutalized call from " - << CS.getParent()->getParent()->getName() << " to " - << F->getName() << "\n"; + DEBUG(dbgs() << "Found devirutalized call from " + << CS.getParent()->getParent()->getName() << " to " + << F->getName() << "\n"); // We now have a direct call where previously we had an indirect call, // so iterate to process this devirtualization site. return true; }; - bool Devirt = any_of(CallHandles, IsDevirtualizedHandle); + bool Devirt = llvm::any_of(CallHandles, IsDevirtualizedHandle); // Rescan to build up a new set of handles and count how many direct // calls remain. If we decide to iterate, this also sets up the input to @@ -753,17 +793,16 @@ public: // Otherwise, if we've already hit our max, we're done. if (Iteration >= MaxIterations) { - if (DebugLogging) - dbgs() << "Found another devirtualization after hitting the max " - "number of repetitions (" - << MaxIterations << ") on SCC: " << *C << "\n"; + DEBUG(dbgs() << "Found another devirtualization after hitting the max " + "number of repetitions (" + << MaxIterations << ") on SCC: " << *C << "\n"); PA.intersect(std::move(PassPA)); break; } - if (DebugLogging) - dbgs() << "Repeating an SCC pass after finding a devirtualization in: " - << *C << "\n"; + DEBUG(dbgs() + << "Repeating an SCC pass after finding a devirtualization in: " + << *C << "\n"); // Move over the new call counts in preparation for iterating. CallCounts = std::move(NewCallCounts); @@ -783,18 +822,19 @@ public: private: PassT Pass; int MaxIterations; - bool DebugLogging; }; /// \brief A function to deduce a function pass type and wrap it in the /// templated adaptor. template <typename PassT> -DevirtSCCRepeatedPass<PassT> -createDevirtSCCRepeatedPass(PassT Pass, int MaxIterations, - bool DebugLogging = false) { - return DevirtSCCRepeatedPass<PassT>(std::move(Pass), MaxIterations, - DebugLogging); -} +DevirtSCCRepeatedPass<PassT> createDevirtSCCRepeatedPass(PassT Pass, + int MaxIterations) { + return DevirtSCCRepeatedPass<PassT>(std::move(Pass), MaxIterations); } -#endif +// Clear out the debug logging macro. +#undef DEBUG_TYPE + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_CGSCCPASSMANAGER_H diff --git a/contrib/llvm/include/llvm/Analysis/CallGraph.h b/contrib/llvm/include/llvm/Analysis/CallGraph.h index 01469a25c96c..c5687def3ebe 100644 --- a/contrib/llvm/include/llvm/Analysis/CallGraph.h +++ b/contrib/llvm/include/llvm/Analysis/CallGraph.h @@ -54,13 +54,17 @@ #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" +#include <cassert> #include <map> +#include <memory> +#include <utility> +#include <vector> namespace llvm { -class Function; -class Module; class CallGraphNode; +class Module; +class raw_ostream; /// \brief The basic data container for the call graph of a \c Module of IR. /// @@ -70,8 +74,8 @@ class CallGraphNode; class CallGraph { Module &M; - typedef std::map<const Function *, std::unique_ptr<CallGraphNode>> - FunctionMapTy; + using FunctionMapTy = + std::map<const Function *, std::unique_ptr<CallGraphNode>>; /// \brief A map from \c Function* to \c CallGraphNode*. FunctionMapTy FunctionMap; @@ -103,8 +107,8 @@ public: void print(raw_ostream &OS) const; void dump() const; - typedef FunctionMapTy::iterator iterator; - typedef FunctionMapTy::const_iterator const_iterator; + using iterator = FunctionMapTy::iterator; + using const_iterator = FunctionMapTy::const_iterator; /// \brief Returns the module the call graph corresponds to. Module &getModule() const { return M; } @@ -162,20 +166,23 @@ class CallGraphNode { public: /// \brief A pair of the calling instruction (a call or invoke) /// and the call graph node being called. - typedef std::pair<WeakTrackingVH, CallGraphNode *> CallRecord; + using CallRecord = std::pair<WeakTrackingVH, CallGraphNode *>; public: - typedef std::vector<CallRecord> CalledFunctionsVector; + using CalledFunctionsVector = std::vector<CallRecord>; /// \brief Creates a node for the specified function. - inline CallGraphNode(Function *F) : F(F), NumReferences(0) {} + inline CallGraphNode(Function *F) : F(F) {} + + CallGraphNode(const CallGraphNode &) = delete; + CallGraphNode &operator=(const CallGraphNode &) = delete; ~CallGraphNode() { assert(NumReferences == 0 && "Node deleted while references remain"); } - typedef std::vector<CallRecord>::iterator iterator; - typedef std::vector<CallRecord>::const_iterator const_iterator; + using iterator = std::vector<CallRecord>::iterator; + using const_iterator = std::vector<CallRecord>::const_iterator; /// \brief Returns the function that this call graph node represents. Function *getFunction() const { return F; } @@ -268,10 +275,7 @@ private: /// \brief The number of times that this CallGraphNode occurs in the /// CalledFunctions array of this or other CallGraphNodes. - unsigned NumReferences; - - CallGraphNode(const CallGraphNode &) = delete; - void operator=(const CallGraphNode &) = delete; + unsigned NumReferences = 0; void DropRef() { --NumReferences; } void AddRef() { ++NumReferences; } @@ -287,11 +291,12 @@ private: /// resulting data. class CallGraphAnalysis : public AnalysisInfoMixin<CallGraphAnalysis> { friend AnalysisInfoMixin<CallGraphAnalysis>; + static AnalysisKey Key; public: - /// \brief A formulaic typedef to inform clients of the result type. - typedef CallGraph Result; + /// \brief A formulaic type to inform clients of the result type. + using Result = CallGraph; /// \brief Compute the \c CallGraph for the module \c M. /// @@ -305,6 +310,7 @@ class CallGraphPrinterPass : public PassInfoMixin<CallGraphPrinterPass> { public: explicit CallGraphPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; @@ -329,8 +335,8 @@ public: const CallGraph &getCallGraph() const { return *G; } CallGraph &getCallGraph() { return *G; } - typedef CallGraph::iterator iterator; - typedef CallGraph::const_iterator const_iterator; + using iterator = CallGraph::iterator; + using const_iterator = CallGraph::const_iterator; /// \brief Returns the module the call graph corresponds to. Module &getModule() const { return G->getModule(); } @@ -399,40 +405,38 @@ public: // Provide graph traits for tranversing call graphs using standard graph // traversals. template <> struct GraphTraits<CallGraphNode *> { - typedef CallGraphNode *NodeRef; - - typedef CallGraphNode::CallRecord CGNPairTy; + using NodeRef = CallGraphNode *; + using CGNPairTy = CallGraphNode::CallRecord; static NodeRef getEntryNode(CallGraphNode *CGN) { return CGN; } - static CallGraphNode *CGNGetValue(CGNPairTy P) { return P.second; } - typedef mapped_iterator<CallGraphNode::iterator, decltype(&CGNGetValue)> - ChildIteratorType; + using ChildIteratorType = + mapped_iterator<CallGraphNode::iterator, decltype(&CGNGetValue)>; static ChildIteratorType child_begin(NodeRef N) { return ChildIteratorType(N->begin(), &CGNGetValue); } + static ChildIteratorType child_end(NodeRef N) { return ChildIteratorType(N->end(), &CGNGetValue); } }; template <> struct GraphTraits<const CallGraphNode *> { - typedef const CallGraphNode *NodeRef; - - typedef CallGraphNode::CallRecord CGNPairTy; + using NodeRef = const CallGraphNode *; + using CGNPairTy = CallGraphNode::CallRecord; static NodeRef getEntryNode(const CallGraphNode *CGN) { return CGN; } - static const CallGraphNode *CGNGetValue(CGNPairTy P) { return P.second; } - typedef mapped_iterator<CallGraphNode::const_iterator, decltype(&CGNGetValue)> - ChildIteratorType; + using ChildIteratorType = + mapped_iterator<CallGraphNode::const_iterator, decltype(&CGNGetValue)>; static ChildIteratorType child_begin(NodeRef N) { return ChildIteratorType(N->begin(), &CGNGetValue); } + static ChildIteratorType child_end(NodeRef N) { return ChildIteratorType(N->end(), &CGNGetValue); } @@ -440,21 +444,25 @@ template <> struct GraphTraits<const CallGraphNode *> { template <> struct GraphTraits<CallGraph *> : public GraphTraits<CallGraphNode *> { + using PairTy = + std::pair<const Function *const, std::unique_ptr<CallGraphNode>>; + static NodeRef getEntryNode(CallGraph *CGN) { return CGN->getExternalCallingNode(); // Start at the external node! } - typedef std::pair<const Function *const, std::unique_ptr<CallGraphNode>> - PairTy; + static CallGraphNode *CGGetValuePtr(const PairTy &P) { return P.second.get(); } // nodes_iterator/begin/end - Allow iteration over all nodes in the graph - typedef mapped_iterator<CallGraph::iterator, decltype(&CGGetValuePtr)> - nodes_iterator; + using nodes_iterator = + mapped_iterator<CallGraph::iterator, decltype(&CGGetValuePtr)>; + static nodes_iterator nodes_begin(CallGraph *CG) { return nodes_iterator(CG->begin(), &CGGetValuePtr); } + static nodes_iterator nodes_end(CallGraph *CG) { return nodes_iterator(CG->end(), &CGGetValuePtr); } @@ -463,26 +471,30 @@ struct GraphTraits<CallGraph *> : public GraphTraits<CallGraphNode *> { template <> struct GraphTraits<const CallGraph *> : public GraphTraits< const CallGraphNode *> { + using PairTy = + std::pair<const Function *const, std::unique_ptr<CallGraphNode>>; + static NodeRef getEntryNode(const CallGraph *CGN) { return CGN->getExternalCallingNode(); // Start at the external node! } - typedef std::pair<const Function *const, std::unique_ptr<CallGraphNode>> - PairTy; + static const CallGraphNode *CGGetValuePtr(const PairTy &P) { return P.second.get(); } // nodes_iterator/begin/end - Allow iteration over all nodes in the graph - typedef mapped_iterator<CallGraph::const_iterator, decltype(&CGGetValuePtr)> - nodes_iterator; + using nodes_iterator = + mapped_iterator<CallGraph::const_iterator, decltype(&CGGetValuePtr)>; + static nodes_iterator nodes_begin(const CallGraph *CG) { return nodes_iterator(CG->begin(), &CGGetValuePtr); } + static nodes_iterator nodes_end(const CallGraph *CG) { return nodes_iterator(CG->end(), &CGGetValuePtr); } }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_CALLGRAPH_H diff --git a/contrib/llvm/include/llvm/Analysis/CallGraphSCCPass.h b/contrib/llvm/include/llvm/Analysis/CallGraphSCCPass.h index f86f64bbb67d..ace54607634c 100644 --- a/contrib/llvm/include/llvm/Analysis/CallGraphSCCPass.h +++ b/contrib/llvm/include/llvm/Analysis/CallGraphSCCPass.h @@ -21,16 +21,16 @@ #ifndef LLVM_ANALYSIS_CALLGRAPHSCCPASS_H #define LLVM_ANALYSIS_CALLGRAPHSCCPASS_H -#include "llvm/Analysis/CallGraph.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/Pass.h" -#include "llvm/PassSupport.h" +#include <vector> namespace llvm { -class CallGraphNode; class CallGraph; -class PMStack; +class CallGraphNode; class CallGraphSCC; +class PMStack; class CallGraphSCCPass : public Pass { public: @@ -38,7 +38,7 @@ public: /// createPrinterPass - Get a pass that prints the Module /// corresponding to a CallGraph. - Pass *createPrinterPass(raw_ostream &O, + Pass *createPrinterPass(raw_ostream &OS, const std::string &Banner) const override; using llvm::Pass::doInitialization; @@ -57,7 +57,6 @@ public: /// /// SCC passes that add or delete functions to the SCC are required to update /// the SCC list, otherwise stale pointers may be dereferenced. - /// virtual bool runOnSCC(CallGraphSCC &SCC) = 0; /// doFinalization - This method is called after the SCC's of the program has @@ -89,7 +88,7 @@ protected: class CallGraphSCC { const CallGraph &CG; // The call graph for this SCC. void *Context; // The CGPassManager object that is vending this. - std::vector<CallGraphNode*> Nodes; + std::vector<CallGraphNode *> Nodes; public: CallGraphSCC(CallGraph &cg, void *context) : CG(cg), Context(context) {} @@ -105,7 +104,8 @@ public: /// Old node has been deleted, and New is to be used in its place. void ReplaceNode(CallGraphNode *Old, CallGraphNode *New); - typedef std::vector<CallGraphNode *>::const_iterator iterator; + using iterator = std::vector<CallGraphNode *>::const_iterator; + iterator begin() const { return Nodes.begin(); } iterator end() const { return Nodes.end(); } @@ -119,16 +119,19 @@ void initializeDummyCGSCCPassPass(PassRegistry &); class DummyCGSCCPass : public CallGraphSCCPass { public: static char ID; + DummyCGSCCPass() : CallGraphSCCPass(ID) { PassRegistry &Registry = *PassRegistry::getPassRegistry(); initializeDummyCGSCCPassPass(Registry); - }; + } + bool runOnSCC(CallGraphSCC &SCC) override { return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); } }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_CALLGRAPHSCCPASS_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/CmpInstAnalysis.h b/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h index 5ec3888d4538..3cc69d9fea29 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/CmpInstAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/CmpInstAnalysis.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TRANSFORMS_UTILS_CMPINSTANALYSIS_H -#define LLVM_TRANSFORMS_UTILS_CMPINSTANALYSIS_H +#ifndef LLVM_ANALYSIS_CMPINSTANALYSIS_H +#define LLVM_ANALYSIS_CMPINSTANALYSIS_H #include "llvm/IR/InstrTypes.h" @@ -60,10 +60,12 @@ namespace llvm { /// equality comparison (which is signless). bool PredicatesFoldable(CmpInst::Predicate p1, CmpInst::Predicate p2); - /// Decompose an icmp into the form ((X & Y) pred Z) if possible. The returned - /// predicate is either == or !=. Returns false if decomposition fails. - bool decomposeBitTestICmp(const ICmpInst *I, CmpInst::Predicate &Pred, - Value *&X, Value *&Y, Value *&Z); + /// Decompose an icmp into the form ((X & Mask) pred 0) if possible. The + /// returned predicate is either == or !=. Returns false if decomposition + /// fails. + bool decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred, + Value *&X, APInt &Mask, + bool LookThroughTrunc = true); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Analysis/ConstantFolding.h b/contrib/llvm/include/llvm/Analysis/ConstantFolding.h index 42034741b8e3..6d4eef412525 100644 --- a/contrib/llvm/include/llvm/Analysis/ConstantFolding.h +++ b/contrib/llvm/include/llvm/Analysis/ConstantFolding.h @@ -79,6 +79,12 @@ ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL); +/// \brief Attempt to constant fold a select instruction with the specified +/// operands. The constant result is returned if successful; if not, null is +/// returned. +Constant *ConstantFoldSelectInstruction(Constant *Cond, Constant *V1, + Constant *V2); + /// \brief Attempt to constant fold a cast with the specified operand. If it /// fails, it returns a constant expression of the specified operand. Constant *ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, @@ -96,6 +102,13 @@ Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, Constant *ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef<unsigned> Idxs); +/// \brief Attempt to constant fold an insertelement instruction with the +/// specified operands and indices. The constant result is returned if +/// successful; if not, null is returned. +Constant *ConstantFoldInsertElementInstruction(Constant *Val, + Constant *Elt, + Constant *Idx); + /// \brief Attempt to constant fold an extractelement instruction with the /// specified operands and indices. The constant result is returned if /// successful; if not, null is returned. diff --git a/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h b/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h index ca50ee2f829a..39f9c39c34e1 100644 --- a/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/contrib/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -30,7 +30,7 @@ struct DefaultAnalysisGraphTraits { template < typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT> > + typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > class DOTGraphTraitsViewer : public FunctionPass { public: DOTGraphTraitsViewer(StringRef GraphName, char &ID) @@ -72,7 +72,7 @@ private: template < typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT> > + typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > class DOTGraphTraitsPrinter : public FunctionPass { public: DOTGraphTraitsPrinter(StringRef GraphName, char &ID) @@ -124,7 +124,7 @@ private: template < typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT> > + typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > class DOTGraphTraitsModuleViewer : public ModulePass { public: DOTGraphTraitsModuleViewer(StringRef GraphName, char &ID) @@ -150,7 +150,7 @@ private: template < typename AnalysisT, bool IsSimple, typename GraphT = AnalysisT *, - typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT> > + typename AnalysisGraphTraitsT = DefaultAnalysisGraphTraits<AnalysisT, GraphT> > class DOTGraphTraitsModulePrinter : public ModulePass { public: DOTGraphTraitsModulePrinter(StringRef GraphName, char &ID) diff --git a/contrib/llvm/include/llvm/Analysis/DemandedBits.h b/contrib/llvm/include/llvm/Analysis/DemandedBits.h index e52c66f361c3..ab8668256ba2 100644 --- a/contrib/llvm/include/llvm/Analysis/DemandedBits.h +++ b/contrib/llvm/include/llvm/Analysis/DemandedBits.h @@ -1,4 +1,4 @@ -//===-- llvm/Analysis/DemandedBits.h - Determine demanded bits --*- C++ -*-===// +//===- llvm/Analysis/DemandedBits.h - Determine demanded bits ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -24,44 +24,45 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace llvm { -class FunctionPass; +class AssumptionCache; +class DominatorTree; class Function; class Instruction; -class DominatorTree; -class AssumptionCache; struct KnownBits; +class raw_ostream; class DemandedBits { public: DemandedBits(Function &F, AssumptionCache &AC, DominatorTree &DT) : - F(F), AC(AC), DT(DT), Analyzed(false) {} + F(F), AC(AC), DT(DT) {} /// Return the bits demanded from instruction I. APInt getDemandedBits(Instruction *I); /// Return true if, during analysis, I could not be reached. bool isInstructionDead(Instruction *I); - + void print(raw_ostream &OS); private: - Function &F; - AssumptionCache &AC; - DominatorTree &DT; - void performAnalysis(); void determineLiveOperandBits(const Instruction *UserI, const Instruction *I, unsigned OperandNo, const APInt &AOut, APInt &AB, KnownBits &Known, KnownBits &Known2); - bool Analyzed; + Function &F; + AssumptionCache &AC; + DominatorTree &DT; + + bool Analyzed = false; // The set of visited instructions (non-integer-typed only). SmallPtrSet<Instruction*, 32> Visited; @@ -71,16 +72,18 @@ private: class DemandedBitsWrapperPass : public FunctionPass { private: mutable Optional<DemandedBits> DB; + public: static char ID; // Pass identification, replacement for typeid + DemandedBitsWrapperPass(); bool runOnFunction(Function &F) override; void getAnalysisUsage(AnalysisUsage &AU) const override; - + /// Clean up memory in between runs void releaseMemory() override; - + DemandedBits &getDemandedBits() { return *DB; } void print(raw_ostream &OS, const Module *M) const override; @@ -89,11 +92,12 @@ public: /// An analysis that produces \c DemandedBits for a function. class DemandedBitsAnalysis : public AnalysisInfoMixin<DemandedBitsAnalysis> { friend AnalysisInfoMixin<DemandedBitsAnalysis>; + static AnalysisKey Key; public: - /// \brief Provide the result typedef for this analysis pass. - typedef DemandedBits Result; + /// \brief Provide the result type for this analysis pass. + using Result = DemandedBits; /// \brief Run the analysis pass over a function and produce demanded bits /// information. @@ -106,12 +110,13 @@ class DemandedBitsPrinterPass : public PassInfoMixin<DemandedBitsPrinterPass> { public: explicit DemandedBitsPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; /// Create a demanded bits analysis pass. FunctionPass *createDemandedBitsWrapperPass(); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_DEMANDED_BITS_H diff --git a/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h b/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h index b566aeaf1fd6..a304dff18c79 100644 --- a/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h +++ b/contrib/llvm/include/llvm/Analysis/DominanceFrontier.h @@ -18,40 +18,46 @@ #ifndef LLVM_ANALYSIS_DOMINANCEFRONTIER_H #define LLVM_ANALYSIS_DOMINANCEFRONTIER_H -#include "llvm/IR/Dominators.h" +#include "llvm/ADT/GraphTraits.h" #include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" +#include "llvm/Support/GenericDomTree.h" +#include <cassert> #include <map> #include <set> +#include <utility> +#include <vector> namespace llvm { +class Function; +class raw_ostream; + //===----------------------------------------------------------------------===// /// DominanceFrontierBase - Common base class for computing forward and inverse /// dominance frontiers for a function. /// template <class BlockT, bool IsPostDom> class DominanceFrontierBase { - public: - typedef std::set<BlockT *> DomSetType; // Dom set for a bb - typedef std::map<BlockT *, DomSetType> DomSetMapType; // Dom set map +public: + using DomSetType = std::set<BlockT *>; // Dom set for a bb + using DomSetMapType = std::map<BlockT *, DomSetType>; // Dom set map protected: - typedef GraphTraits<BlockT *> BlockTraits; + using BlockTraits = GraphTraits<BlockT *>; DomSetMapType Frontiers; - std::vector<BlockT *> Roots; + // Postdominators can have multiple roots. + SmallVector<BlockT *, IsPostDom ? 4 : 1> Roots; static constexpr bool IsPostDominators = IsPostDom; - public: - DominanceFrontierBase() {} +public: + DominanceFrontierBase() = default; /// getRoots - Return the root blocks of the current CFG. This may include /// multiple blocks if we are computing post dominators. For forward /// dominators, this will always be a single block (the entry node). - /// - inline const std::vector<BlockT *> &getRoots() const { - return Roots; - } + const SmallVectorImpl<BlockT *> &getRoots() const { return Roots; } BlockT *getRoot() const { assert(Roots.size() == 1 && "Should always have entry node!"); @@ -59,7 +65,6 @@ protected: } /// isPostDominator - Returns true if analysis based of postdoms - /// bool isPostDominator() const { return IsPostDominators; } @@ -69,8 +74,9 @@ protected: } // Accessor interface: - typedef typename DomSetMapType::iterator iterator; - typedef typename DomSetMapType::const_iterator const_iterator; + using iterator = typename DomSetMapType::iterator; + using const_iterator = typename DomSetMapType::const_iterator; + iterator begin() { return Frontiers.begin(); } const_iterator begin() const { return Frontiers.begin(); } iterator end() { return Frontiers.end(); } @@ -115,19 +121,19 @@ protected: template <class BlockT> class ForwardDominanceFrontierBase : public DominanceFrontierBase<BlockT, false> { - private: - typedef GraphTraits<BlockT *> BlockTraits; +private: + using BlockTraits = GraphTraits<BlockT *>; public: - typedef DomTreeBase<BlockT> DomTreeT; - typedef DomTreeNodeBase<BlockT> DomTreeNodeT; - typedef typename DominanceFrontierBase<BlockT, false>::DomSetType DomSetType; - - void analyze(DomTreeT &DT) { - this->Roots = DT.getRoots(); - assert(this->Roots.size() == 1 && - "Only one entry block for forward domfronts!"); - calculate(DT, DT[this->Roots[0]]); + using DomTreeT = DomTreeBase<BlockT>; + using DomTreeNodeT = DomTreeNodeBase<BlockT>; + using DomSetType = typename DominanceFrontierBase<BlockT, false>::DomSetType; + + void analyze(DomTreeT &DT) { + assert(DT.getRoots().size() == 1 && + "Only one entry block for forward domfronts!"); + this->Roots = {DT.getRoot()}; + calculate(DT, DT[this->Roots[0]]); } const DomSetType &calculate(const DomTreeT &DT, const DomTreeNodeT *Node); @@ -135,20 +141,21 @@ public: class DominanceFrontier : public ForwardDominanceFrontierBase<BasicBlock> { public: - typedef DomTreeBase<BasicBlock> DomTreeT; - typedef DomTreeNodeBase<BasicBlock> DomTreeNodeT; - typedef DominanceFrontierBase<BasicBlock, false>::DomSetType DomSetType; - typedef DominanceFrontierBase<BasicBlock, false>::iterator iterator; - typedef DominanceFrontierBase<BasicBlock, false>::const_iterator - const_iterator; - - /// Handle invalidation explicitly. - bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &); + using DomTreeT = DomTreeBase<BasicBlock>; + using DomTreeNodeT = DomTreeNodeBase<BasicBlock>; + using DomSetType = DominanceFrontierBase<BasicBlock, false>::DomSetType; + using iterator = DominanceFrontierBase<BasicBlock, false>::iterator; + using const_iterator = + DominanceFrontierBase<BasicBlock, false>::const_iterator; + + /// Handle invalidation explicitly. + bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &); }; class DominanceFrontierWrapperPass : public FunctionPass { DominanceFrontier DF; + public: static char ID; // Pass ID, replacement for typeid @@ -176,11 +183,12 @@ extern template class ForwardDominanceFrontierBase<BasicBlock>; class DominanceFrontierAnalysis : public AnalysisInfoMixin<DominanceFrontierAnalysis> { friend AnalysisInfoMixin<DominanceFrontierAnalysis>; + static AnalysisKey Key; public: - /// \brief Provide the result typedef for this analysis pass. - typedef DominanceFrontier Result; + /// \brief Provide the result type for this analysis pass. + using Result = DominanceFrontier; /// \brief Run the analysis pass over a function and produce a dominator tree. DominanceFrontier run(Function &F, FunctionAnalysisManager &AM); @@ -193,9 +201,10 @@ class DominanceFrontierPrinterPass public: explicit DominanceFrontierPrinterPass(raw_ostream &OS); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_DOMINANCEFRONTIER_H diff --git a/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h b/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h index 5093b975e709..dffb2e02b621 100644 --- a/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h +++ b/contrib/llvm/include/llvm/Analysis/DominanceFrontierImpl.h @@ -18,21 +18,28 @@ #ifndef LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H #define LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H +#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GenericDomTree.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <set> +#include <utility> +#include <vector> namespace llvm { template <class BlockT> class DFCalculateWorkObject { public: - typedef DomTreeNodeBase<BlockT> DomTreeNodeT; + using DomTreeNodeT = DomTreeNodeBase<BlockT>; DFCalculateWorkObject(BlockT *B, BlockT *P, const DomTreeNodeT *N, const DomTreeNodeT *PN) : currentBB(B), parentBB(P), Node(N), parentNode(PN) {} + BlockT *currentBB; BlockT *parentBB; const DomTreeNodeT *Node; @@ -219,6 +226,6 @@ ForwardDominanceFrontierBase<BlockT>::calculate(const DomTreeT &DT, return *Result; } -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_DOMINANCEFRONTIERIMPL_H diff --git a/contrib/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h b/contrib/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h index 007e4d8602fa..8b1c10139de8 100644 --- a/contrib/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/IndirectCallPromotionAnalysis.h @@ -27,10 +27,12 @@ private: // Allocate space to read the profile annotation. std::unique_ptr<InstrProfValueData[]> ValueDataArray; - // Count is the call count for the direct-call target and - // TotalCount is the call count for the indirect-call callsite. + // Count is the call count for the direct-call target. + // TotalCount is the total call count for the indirect-call callsite. + // RemainingCount is the TotalCount minus promoted-direct-call count. // Return true we should promote this indirect-call target. - bool isPromotionProfitable(uint64_t Count, uint64_t TotalCount); + bool isPromotionProfitable(uint64_t Count, uint64_t TotalCount, + uint64_t RemainingCount); // Returns the number of profitable candidates to promote for the // current ValueDataArray and the given \p Inst. diff --git a/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h b/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h index 3c40cc0235cc..dde56a143c51 100644 --- a/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h +++ b/contrib/llvm/include/llvm/Analysis/IndirectCallSiteVisitor.h @@ -27,7 +27,7 @@ struct PGOIndirectCallSiteVisitor }; // Helper function that finds all indirect call sites. -static inline std::vector<Instruction *> findIndirectCallSites(Function &F) { +inline std::vector<Instruction *> findIndirectCallSites(Function &F) { PGOIndirectCallSiteVisitor ICV; ICV.visit(F); return ICV.IndirectCallInsts; diff --git a/contrib/llvm/include/llvm/Analysis/InlineCost.h b/contrib/llvm/include/llvm/Analysis/InlineCost.h index f33a2de5a5f4..985f3880ed3a 100644 --- a/contrib/llvm/include/llvm/Analysis/InlineCost.h +++ b/contrib/llvm/include/llvm/Analysis/InlineCost.h @@ -16,6 +16,7 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include <cassert> #include <climits> @@ -105,6 +106,12 @@ public: return Cost; } + /// \brief Get the threshold against which the cost was computed + int getThreshold() const { + assert(isVariable() && "Invalid access of InlineCost"); + return Threshold; + } + /// \brief Get the cost delta from the threshold for inlining. /// Only valid if the cost is of the variable kind. Returns a negative /// value if the cost is too high to inline. @@ -139,8 +146,15 @@ struct InlineParams { /// Threshold to use when the callsite is considered hot. Optional<int> HotCallSiteThreshold; + /// Threshold to use when the callsite is considered hot relative to function + /// entry. + Optional<int> LocallyHotCallSiteThreshold; + /// Threshold to use when the callsite is considered cold. Optional<int> ColdCallSiteThreshold; + + /// Compute inline cost even when the cost has exceeded the threshold. + Optional<bool> ComputeFullInlineCost; }; /// Generate the parameters to tune the inline cost analysis based only on the @@ -175,12 +189,11 @@ int getCallsiteCost(CallSite CS, const DataLayout &DL); /// /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. -InlineCost -getInlineCost(CallSite CS, const InlineParams &Params, - TargetTransformInfo &CalleeTTI, - std::function<AssumptionCache &(Function &)> &GetAssumptionCache, - Optional<function_ref<BlockFrequencyInfo &(Function &)>> GetBFI, - ProfileSummaryInfo *PSI); +InlineCost getInlineCost( + CallSite CS, const InlineParams &Params, TargetTransformInfo &CalleeTTI, + std::function<AssumptionCache &(Function &)> &GetAssumptionCache, + Optional<function_ref<BlockFrequencyInfo &(Function &)>> GetBFI, + ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE = nullptr); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -192,7 +205,7 @@ getInlineCost(CallSite CS, Function *Callee, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function<AssumptionCache &(Function &)> &GetAssumptionCache, Optional<function_ref<BlockFrequencyInfo &(Function &)>> GetBFI, - ProfileSummaryInfo *PSI); + ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); diff --git a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h index be0f32ef444a..3932a2ec2498 100644 --- a/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/contrib/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -161,6 +161,10 @@ Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, Value *SimplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q); +/// Given operands for an InsertElement, fold the result or return null. +Value *SimplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, + const SimplifyQuery &Q); + /// Given operands for an ExtractValueInst, fold the result or return null. Value *SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q); diff --git a/contrib/llvm/include/llvm/Analysis/Interval.h b/contrib/llvm/include/llvm/Analysis/Interval.h index a63a004043cc..f3714dddedd5 100644 --- a/contrib/llvm/include/llvm/Analysis/Interval.h +++ b/contrib/llvm/include/llvm/Analysis/Interval.h @@ -39,10 +39,11 @@ class Interval { /// interval. Also, any loops in this interval must go through the HeaderNode. /// BasicBlock *HeaderNode; + public: - typedef std::vector<BasicBlock*>::iterator succ_iterator; - typedef std::vector<BasicBlock*>::iterator pred_iterator; - typedef std::vector<BasicBlock*>::iterator node_iterator; + using succ_iterator = std::vector<BasicBlock*>::iterator; + using pred_iterator = std::vector<BasicBlock*>::iterator; + using node_iterator = std::vector<BasicBlock*>::iterator; inline Interval(BasicBlock *Header) : HeaderNode(Header) { Nodes.push_back(Header); @@ -51,18 +52,15 @@ public: inline BasicBlock *getHeaderNode() const { return HeaderNode; } /// Nodes - The basic blocks in this interval. - /// std::vector<BasicBlock*> Nodes; /// Successors - List of BasicBlocks that are reachable directly from nodes in /// this interval, but are not in the interval themselves. /// These nodes necessarily must be header nodes for other intervals. - /// std::vector<BasicBlock*> Successors; /// Predecessors - List of BasicBlocks that have this Interval's header block /// as one of their successors. - /// std::vector<BasicBlock*> Predecessors; /// contains - Find out if a basic block is in this interval @@ -88,7 +86,6 @@ public: /// Equality operator. It is only valid to compare two intervals from the /// same partition, because of this, all we have to check is the header node /// for equality. - /// inline bool operator==(const Interval &I) const { return HeaderNode == I.HeaderNode; } @@ -121,8 +118,8 @@ inline Interval::pred_iterator pred_end(Interval *I) { } template <> struct GraphTraits<Interval*> { - typedef Interval *NodeRef; - typedef Interval::succ_iterator ChildIteratorType; + using NodeRef = Interval *; + using ChildIteratorType = Interval::succ_iterator; static NodeRef getEntryNode(Interval *I) { return I; } @@ -131,14 +128,15 @@ template <> struct GraphTraits<Interval*> { static ChildIteratorType child_end(NodeRef N) { return succ_end(N); } }; -template <> struct GraphTraits<Inverse<Interval*> > { - typedef Interval *NodeRef; - typedef Interval::pred_iterator ChildIteratorType; +template <> struct GraphTraits<Inverse<Interval*>> { + using NodeRef = Interval *; + using ChildIteratorType = Interval::pred_iterator; + static NodeRef getEntryNode(Inverse<Interval *> G) { return G.Graph; } static ChildIteratorType child_begin(NodeRef N) { return pred_begin(N); } static ChildIteratorType child_end(NodeRef N) { return pred_end(N); } }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_INTERVAL_H diff --git a/contrib/llvm/include/llvm/Analysis/IntervalIterator.h b/contrib/llvm/include/llvm/Analysis/IntervalIterator.h index 655ce2dab413..6ffcae592e98 100644 --- a/contrib/llvm/include/llvm/Analysis/IntervalIterator.h +++ b/contrib/llvm/include/llvm/Analysis/IntervalIterator.h @@ -33,26 +33,32 @@ #ifndef LLVM_ANALYSIS_INTERVALITERATOR_H #define LLVM_ANALYSIS_INTERVALITERATOR_H +#include "llvm/ADT/GraphTraits.h" +#include "llvm/Analysis/Interval.h" #include "llvm/Analysis/IntervalPartition.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Function.h" +#include "llvm/Support/ErrorHandling.h" #include <algorithm> +#include <cassert> +#include <iterator> #include <set> +#include <utility> #include <vector> namespace llvm { +class BasicBlock; + // getNodeHeader - Given a source graph node and the source graph, return the // BasicBlock that is the header node. This is the opposite of // getSourceGraphNode. -// inline BasicBlock *getNodeHeader(BasicBlock *BB) { return BB; } inline BasicBlock *getNodeHeader(Interval *I) { return I->getHeaderNode(); } // getSourceGraphNode - Given a BasicBlock and the source graph, return the // source graph node that corresponds to the BasicBlock. This is the opposite // of getNodeHeader. -// inline BasicBlock *getSourceGraphNode(Function *, BasicBlock *BB) { return BB; } @@ -64,7 +70,6 @@ inline Interval *getSourceGraphNode(IntervalPartition *IP, BasicBlock *BB) { // with the task of adding a node to the new interval, depending on the // type of the source node. In the case of a CFG source graph (BasicBlock // case), the BasicBlock itself is added to the interval. -// inline void addNodeToInterval(Interval *Int, BasicBlock *BB) { Int->Nodes.push_back(BB); } @@ -75,28 +80,25 @@ inline void addNodeToInterval(Interval *Int, BasicBlock *BB) { // case), the BasicBlock itself is added to the interval. In the case of // an IntervalPartition source graph (Interval case), all of the member // BasicBlocks are added to the interval. -// inline void addNodeToInterval(Interval *Int, Interval *I) { // Add all of the nodes in I as new nodes in Int. Int->Nodes.insert(Int->Nodes.end(), I->Nodes.begin(), I->Nodes.end()); } - - - - -template<class NodeTy, class OrigContainer_t, class GT = GraphTraits<NodeTy*>, - class IGT = GraphTraits<Inverse<NodeTy*> > > +template<class NodeTy, class OrigContainer_t, class GT = GraphTraits<NodeTy *>, + class IGT = GraphTraits<Inverse<NodeTy *>>> class IntervalIterator { - std::vector<std::pair<Interval*, typename Interval::succ_iterator> > IntStack; - std::set<BasicBlock*> Visited; + std::vector<std::pair<Interval *, typename Interval::succ_iterator>> IntStack; + std::set<BasicBlock *> Visited; OrigContainer_t *OrigContainer; bool IOwnMem; // If True, delete intervals when done with them // See file header for conditions of use + public: - typedef std::forward_iterator_tag iterator_category; + using iterator_category = std::forward_iterator_tag; + + IntervalIterator() = default; // End iterator, empty stack - IntervalIterator() {} // End iterator, empty stack IntervalIterator(Function *M, bool OwnMemory) : IOwnMem(OwnMemory) { OrigContainer = M; if (!ProcessInterval(&M->front())) { @@ -157,6 +159,7 @@ public: return *this; } + IntervalIterator operator++(int) { // Postincrement IntervalIterator tmp = *this; ++*this; @@ -171,7 +174,6 @@ private: // // This method is templated because it may operate on two different source // graphs: a basic block graph, or a preexisting interval graph. - // bool ProcessInterval(NodeTy *Node) { BasicBlock *Header = getNodeHeader(Node); if (!Visited.insert(Header).second) @@ -196,7 +198,6 @@ private: // // This method is templated because it may operate on two different source // graphs: a basic block graph, or a preexisting interval graph. - // void ProcessNode(Interval *Int, NodeTy *Node) { assert(Int && "Null interval == bad!"); assert(Node && "Null Node == bad!"); @@ -241,10 +242,9 @@ private: } }; -typedef IntervalIterator<BasicBlock, Function> function_interval_iterator; -typedef IntervalIterator<Interval, IntervalPartition> - interval_part_interval_iterator; - +using function_interval_iterator = IntervalIterator<BasicBlock, Function>; +using interval_part_interval_iterator = + IntervalIterator<Interval, IntervalPartition>; inline function_interval_iterator intervals_begin(Function *F, bool DeleteInts = true) { @@ -263,6 +263,6 @@ inline interval_part_interval_iterator intervals_end(IntervalPartition &IP) { return interval_part_interval_iterator(); } -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_INTERVALITERATOR_H diff --git a/contrib/llvm/include/llvm/Analysis/IntervalPartition.h b/contrib/llvm/include/llvm/Analysis/IntervalPartition.h index 274be2bdcfa9..50335165711f 100644 --- a/contrib/llvm/include/llvm/Analysis/IntervalPartition.h +++ b/contrib/llvm/include/llvm/Analysis/IntervalPartition.h @@ -23,12 +23,15 @@ #ifndef LLVM_ANALYSIS_INTERVALPARTITION_H #define LLVM_ANALYSIS_INTERVALPARTITION_H -#include "llvm/Analysis/Interval.h" #include "llvm/Pass.h" #include <map> +#include <vector> namespace llvm { +class BasicBlock; +class Interval; + //===----------------------------------------------------------------------===// // // IntervalPartition - This class builds and holds an "interval partition" for @@ -38,17 +41,17 @@ namespace llvm { // nodes following it. // class IntervalPartition : public FunctionPass { - typedef std::map<BasicBlock*, Interval*> IntervalMapTy; + using IntervalMapTy = std::map<BasicBlock *, Interval *>; IntervalMapTy IntervalMap; - typedef std::vector<Interval*> IntervalListTy; - Interval *RootInterval; - std::vector<Interval*> Intervals; + using IntervalListTy = std::vector<Interval *>; + Interval *RootInterval = nullptr; + std::vector<Interval *> Intervals; public: static char ID; // Pass identification, replacement for typeid - IntervalPartition() : FunctionPass(ID), RootInterval(nullptr) { + IntervalPartition() : FunctionPass(ID) { initializeIntervalPartitionPass(*PassRegistry::getPassRegistry()); } @@ -58,7 +61,6 @@ public: // IntervalPartition ctor - Build a reduced interval partition from an // existing interval graph. This takes an additional boolean parameter to // distinguish it from a copy constructor. Always pass in false for now. - // IntervalPartition(IntervalPartition &I, bool); // print - Show contents in human readable format... @@ -95,17 +97,15 @@ private: // addIntervalToPartition - Add an interval to the internal list of intervals, // and then add mappings from all of the basic blocks in the interval to the // interval itself (in the IntervalMap). - // void addIntervalToPartition(Interval *I); // updatePredecessors - Interval generation only sets the successor fields of // the interval data structures. After interval generation is complete, // run through all of the intervals and propagate successor info as // predecessor info. - // void updatePredecessors(Interval *Int); }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_INTERVALPARTITION_H diff --git a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h index a025f2275fb4..d1ec6a9dcc55 100644 --- a/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h +++ b/contrib/llvm/include/llvm/Analysis/LazyCallGraph.h @@ -35,28 +35,33 @@ #ifndef LLVM_ANALYSIS_LAZYCALLGRAPH_H #define LLVM_ANALYSIS_LAZYCALLGRAPH_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/PointerUnion.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/raw_ostream.h" +#include <cassert> #include <iterator> +#include <string> #include <utility> namespace llvm { -class PreservedAnalyses; -class raw_ostream; + +class Module; +class Value; /// A lazily constructed view of the call graph of a module. /// @@ -183,8 +188,8 @@ public: friend class LazyCallGraph::Node; friend class LazyCallGraph::RefSCC; - typedef SmallVector<Edge, 4> VectorT; - typedef SmallVectorImpl<Edge> VectorImplT; + using VectorT = SmallVector<Edge, 4>; + using VectorImplT = SmallVectorImpl<Edge>; public: /// An iterator used for the edges to both entry nodes and child nodes. @@ -204,7 +209,7 @@ public: } public: - iterator() {} + iterator() = default; using iterator_adaptor_base::operator++; iterator &operator++() { @@ -240,7 +245,7 @@ public: } public: - call_iterator() {} + call_iterator() = default; using iterator_adaptor_base::operator++; call_iterator &operator++() { @@ -256,11 +261,17 @@ public: Edge &operator[](int i) { return Edges[i]; } Edge &operator[](Node &N) { assert(EdgeIndexMap.find(&N) != EdgeIndexMap.end() && "No such edge!"); - return Edges[EdgeIndexMap.find(&N)->second]; + auto &E = Edges[EdgeIndexMap.find(&N)->second]; + assert(E && "Dead or null edge!"); + return E; } + Edge *lookup(Node &N) { auto EI = EdgeIndexMap.find(&N); - return EI != EdgeIndexMap.end() ? &Edges[EI->second] : nullptr; + if (EI == EdgeIndexMap.end()) + return nullptr; + auto &E = Edges[EI->second]; + return E ? &E : nullptr; } call_iterator call_begin() { @@ -329,7 +340,18 @@ public: bool operator!=(const Node &N) const { return !operator==(N); } /// Tests whether the node has been populated with edges. - operator bool() const { return Edges.hasValue(); } + bool isPopulated() const { return Edges.hasValue(); } + + /// Tests whether this is actually a dead node and no longer valid. + /// + /// Users rarely interact with nodes in this state and other methods are + /// invalid. This is used to model a node in an edge list where the + /// function has been completely removed. + bool isDead() const { + assert(!G == !F && + "Both graph and function pointers should be null or non-null."); + return !G; + } // We allow accessing the edges by dereferencing or using the arrow // operator, essentially wrapping the internal optional. @@ -365,15 +387,14 @@ public: // We provide for the DFS numbering and Tarjan walk lowlink numbers to be // stored directly within the node. These are both '-1' when nodes are part // of an SCC (or RefSCC), or '0' when not yet reached in a DFS walk. - int DFSNumber; - int LowLink; + int DFSNumber = 0; + int LowLink = 0; Optional<EdgeSequence> Edges; /// Basic constructor implements the scanning of F into Edges and /// EdgeIndexMap. - Node(LazyCallGraph &G, Function &F) - : G(&G), F(&F), DFSNumber(0), LowLink(0) {} + Node(LazyCallGraph &G, Function &F) : G(&G), F(&F) {} /// Implementation of the scan when populating. EdgeSequence &populateSlow(); @@ -462,7 +483,7 @@ public: #endif public: - typedef pointee_iterator<SmallVectorImpl<Node *>::const_iterator> iterator; + using iterator = pointee_iterator<SmallVectorImpl<Node *>::const_iterator>; iterator begin() const { return Nodes.begin(); } iterator end() const { return Nodes.end(); } @@ -528,7 +549,6 @@ public: friend class LazyCallGraph::Node; LazyCallGraph *G; - SmallPtrSet<RefSCC *, 1> Parents; /// A postorder list of the inner SCCs. SmallVector<SCC *, 4> SCCs; @@ -541,7 +561,6 @@ public: RefSCC(LazyCallGraph &G); void clear() { - Parents.clear(); SCCs.clear(); SCCIndices.clear(); } @@ -592,10 +611,10 @@ public: void handleTrivialEdgeInsertion(Node &SourceN, Node &TargetN); public: - typedef pointee_iterator<SmallVectorImpl<SCC *>::const_iterator> iterator; - typedef iterator_range<iterator> range; - typedef pointee_iterator<SmallPtrSetImpl<RefSCC *>::const_iterator> - parent_iterator; + using iterator = pointee_iterator<SmallVectorImpl<SCC *>::const_iterator>; + using range = iterator_range<iterator>; + using parent_iterator = + pointee_iterator<SmallPtrSetImpl<RefSCC *>::const_iterator>; iterator begin() const { return SCCs.begin(); } iterator end() const { return SCCs.end(); } @@ -608,27 +627,34 @@ public: return SCCs.begin() + SCCIndices.find(&C)->second; } - parent_iterator parent_begin() const { return Parents.begin(); } - parent_iterator parent_end() const { return Parents.end(); } - - iterator_range<parent_iterator> parents() const { - return make_range(parent_begin(), parent_end()); - } + /// Test if this RefSCC is a parent of \a RC. + /// + /// CAUTION: This method walks every edge in the \c RefSCC, it can be very + /// expensive. + bool isParentOf(const RefSCC &RC) const; - /// Test if this RefSCC is a parent of \a C. - bool isParentOf(const RefSCC &C) const { return C.isChildOf(*this); } + /// Test if this RefSCC is an ancestor of \a RC. + /// + /// CAUTION: This method walks the directed graph of edges as far as + /// necessary to find a possible path to the argument. In the worst case + /// this may walk the entire graph and can be extremely expensive. + bool isAncestorOf(const RefSCC &RC) const; - /// Test if this RefSCC is an ancestor of \a C. - bool isAncestorOf(const RefSCC &C) const { return C.isDescendantOf(*this); } + /// Test if this RefSCC is a child of \a RC. + /// + /// CAUTION: This method walks every edge in the argument \c RefSCC, it can + /// be very expensive. + bool isChildOf(const RefSCC &RC) const { return RC.isParentOf(*this); } - /// Test if this RefSCC is a child of \a C. - bool isChildOf(const RefSCC &C) const { - return Parents.count(const_cast<RefSCC *>(&C)); + /// Test if this RefSCC is a descendant of \a RC. + /// + /// CAUTION: This method walks the directed graph of edges as far as + /// necessary to find a possible path from the argument. In the worst case + /// this may walk the entire graph and can be extremely expensive. + bool isDescendantOf(const RefSCC &RC) const { + return RC.isAncestorOf(*this); } - /// Test if this RefSCC is a descendant of \a C. - bool isDescendantOf(const RefSCC &C) const; - /// Provide a short name by printing this RefSCC to a std::string. /// /// This copes with the fact that we don't have a name per-se for an RefSCC @@ -774,26 +800,25 @@ public: /// though, so be careful calling this while iterating over them. void removeOutgoingEdge(Node &SourceN, Node &TargetN); - /// Remove a ref edge which is entirely within this RefSCC. + /// Remove a list of ref edges which are entirely within this RefSCC. /// - /// Both the \a SourceN and the \a TargetN must be within this RefSCC. - /// Removing such an edge may break cycles that form this RefSCC and thus - /// this operation may change the RefSCC graph significantly. In + /// Both the \a SourceN and all of the \a TargetNs must be within this + /// RefSCC. Removing these edges may break cycles that form this RefSCC and + /// thus this operation may change the RefSCC graph significantly. In /// particular, this operation will re-form new RefSCCs based on the /// remaining connectivity of the graph. The following invariants are /// guaranteed to hold after calling this method: /// - /// 1) This RefSCC is still a RefSCC in the graph. - /// 2) This RefSCC will be the parent of any new RefSCCs. Thus, this RefSCC - /// is preserved as the root of any new RefSCC DAG formed. - /// 3) No RefSCC other than this RefSCC has its member set changed (this is + /// 1) If a ref-cycle remains after removal, it leaves this RefSCC intact + /// and in the graph. No new RefSCCs are built. + /// 2) Otherwise, this RefSCC will be dead after this call and no longer in + /// the graph or the postorder traversal of the call graph. Any iterator + /// pointing at this RefSCC will become invalid. + /// 3) All newly formed RefSCCs will be returned and the order of the + /// RefSCCs returned will be a valid postorder traversal of the new + /// RefSCCs. + /// 4) No RefSCC other than this RefSCC has its member set changed (this is /// inherent in the definition of removing such an edge). - /// 4) All of the parent links of the RefSCC graph will be updated to - /// reflect the new RefSCC structure. - /// 5) All RefSCCs formed out of this RefSCC, excluding this RefSCC, will - /// be returned in post-order. - /// 6) The order of the RefSCCs in the vector will be a valid postorder - /// traversal of the new RefSCCs. /// /// These invariants are very important to ensure that we can build /// optimization pipelines on top of the CGSCC pass manager which @@ -812,11 +837,9 @@ public: /// within this RefSCC and edges from this RefSCC to child RefSCCs. Some /// effort has been made to minimize the overhead of common cases such as /// self-edges and edge removals which result in a spanning tree with no - /// more cycles. There are also detailed comments within the implementation - /// on techniques which could substantially improve this routine's - /// efficiency. + /// more cycles. SmallVector<RefSCC *, 1> removeInternalRefEdge(Node &SourceN, - Node &TargetN); + ArrayRef<Node *> TargetNs); /// A convenience wrapper around the above to handle trivial cases of /// inserting a new call edge. @@ -870,14 +893,13 @@ public: struct IsAtEndT {}; LazyCallGraph *G; - RefSCC *RC; + RefSCC *RC = nullptr; /// Build the begin iterator for a node. postorder_ref_scc_iterator(LazyCallGraph &G) : G(&G), RC(getRC(G, 0)) {} /// Build the end iterator for a node. This is selected purely by overload. - postorder_ref_scc_iterator(LazyCallGraph &G, IsAtEndT /*Nonce*/) - : G(&G), RC(nullptr) {} + postorder_ref_scc_iterator(LazyCallGraph &G, IsAtEndT /*Nonce*/) : G(&G) {} /// Get the post-order RefSCC at the given index of the postorder walk, /// populating it if necessary. @@ -1079,8 +1101,8 @@ public: ///@} private: - typedef SmallVectorImpl<Node *>::reverse_iterator node_stack_iterator; - typedef iterator_range<node_stack_iterator> node_stack_range; + using node_stack_iterator = SmallVectorImpl<Node *>::reverse_iterator; + using node_stack_range = iterator_range<node_stack_iterator>; /// Allocator that holds all the call graph nodes. SpecificBumpPtrAllocator<Node> BPA; @@ -1112,11 +1134,6 @@ private: /// RefSCCs. DenseMap<RefSCC *, int> RefSCCIndices; - /// The leaf RefSCCs of the graph. - /// - /// These are all of the RefSCCs which have no children. - SmallVector<RefSCC *, 4> LeafRefSCCs; - /// Defined functions that are also known library functions which the /// optimizer can reason about and therefore might introduce calls to out of /// thin air. @@ -1163,12 +1180,6 @@ private: /// Build the SCCs for a RefSCC out of a list of nodes. void buildSCCs(RefSCC &RC, node_stack_range Nodes); - /// Connect a RefSCC into the larger graph. - /// - /// This walks the edges to connect the RefSCC to its children's parent set, - /// and updates the root leaf list. - void connectRefSCC(RefSCC &RC); - /// Get the index of a RefSCC within the postorder traversal. /// /// Requires that this RefSCC is a valid one in the (perhaps partial) @@ -1185,7 +1196,9 @@ private: inline LazyCallGraph::Edge::Edge() : Value() {} inline LazyCallGraph::Edge::Edge(Node &N, Kind K) : Value(&N, K) {} -inline LazyCallGraph::Edge::operator bool() const { return Value.getPointer(); } +inline LazyCallGraph::Edge::operator bool() const { + return Value.getPointer() && !Value.getPointer()->isDead(); +} inline LazyCallGraph::Edge::Kind LazyCallGraph::Edge::getKind() const { assert(*this && "Queried a null edge!"); @@ -1209,16 +1222,16 @@ inline Function &LazyCallGraph::Edge::getFunction() const { // Provide GraphTraits specializations for call graphs. template <> struct GraphTraits<LazyCallGraph::Node *> { - typedef LazyCallGraph::Node *NodeRef; - typedef LazyCallGraph::EdgeSequence::iterator ChildIteratorType; + using NodeRef = LazyCallGraph::Node *; + using ChildIteratorType = LazyCallGraph::EdgeSequence::iterator; static NodeRef getEntryNode(NodeRef N) { return N; } static ChildIteratorType child_begin(NodeRef N) { return (*N)->begin(); } static ChildIteratorType child_end(NodeRef N) { return (*N)->end(); } }; template <> struct GraphTraits<LazyCallGraph *> { - typedef LazyCallGraph::Node *NodeRef; - typedef LazyCallGraph::EdgeSequence::iterator ChildIteratorType; + using NodeRef = LazyCallGraph::Node *; + using ChildIteratorType = LazyCallGraph::EdgeSequence::iterator; static NodeRef getEntryNode(NodeRef N) { return N; } static ChildIteratorType child_begin(NodeRef N) { return (*N)->begin(); } @@ -1228,11 +1241,12 @@ template <> struct GraphTraits<LazyCallGraph *> { /// An analysis pass which computes the call graph for a module. class LazyCallGraphAnalysis : public AnalysisInfoMixin<LazyCallGraphAnalysis> { friend AnalysisInfoMixin<LazyCallGraphAnalysis>; + static AnalysisKey Key; public: /// Inform generic clients of the result type. - typedef LazyCallGraph Result; + using Result = LazyCallGraph; /// Compute the \c LazyCallGraph for the module \c M. /// @@ -1268,6 +1282,7 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_LAZYCALLGRAPH_H diff --git a/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 2568903c57f3..54f151ef82e2 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -163,7 +163,7 @@ public: }; MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) - : PSE(PSE), InnermostLoop(L), AccessIdx(0), + : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeRegisterWidth(-1U), ShouldRetryWithRuntimeCheck(false), SafeForVectorization(true), RecordDependences(true) {} @@ -199,6 +199,10 @@ public: /// the accesses safely with. uint64_t getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; } + /// \brief Return the number of elements that are safe to operate on + /// simultaneously, multiplied by the size of the element in bits. + uint64_t getMaxSafeRegisterWidth() const { return MaxSafeRegisterWidth; } + /// \brief In same cases when the dependency check fails we can still /// vectorize the loop with a dynamic array access check. bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } @@ -255,6 +259,12 @@ private: // We can access this many bytes in parallel safely. uint64_t MaxSafeDepDistBytes; + /// \brief Number of elements (from consecutive iterations) that are safe to + /// operate on simultaneously, multiplied by the size of the element in bits. + /// The size of the element is taken from the memory access that is most + /// restrictive. + uint64_t MaxSafeRegisterWidth; + /// \brief If we see a non-constant dependence distance we can still try to /// vectorize this loop with runtime checks. bool ShouldRetryWithRuntimeCheck; @@ -657,6 +667,21 @@ int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp, const ValueToValueMap &StridesMap = ValueToValueMap(), bool Assume = false, bool ShouldCheckWrap = true); +/// \brief Attempt to sort the 'loads' in \p VL and return the sorted values in +/// \p Sorted. +/// +/// Returns 'false' if sorting is not legal or feasible, otherwise returns +/// 'true'. If \p Mask is not null, it also returns the \p Mask which is the +/// shuffle mask for actual memory access order. +/// +/// For example, for a given VL of memory accesses in program order, a[i+2], +/// a[i+0], a[i+1] and a[i+3], this function will sort the VL and save the +/// sorted value in 'Sorted' as a[i+0], a[i+1], a[i+2], a[i+3] and saves the +/// mask for actual memory accesses in program order in 'Mask' as <2,0,1,3> +bool sortLoadAccesses(ArrayRef<Value *> VL, const DataLayout &DL, + ScalarEvolution &SE, SmallVectorImpl<Value *> &Sorted, + SmallVectorImpl<unsigned> *Mask = nullptr); + /// \brief Returns true if the memory operations \p A and \p B are consecutive. /// This is a simple API that does not depend on the analysis pass. bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, diff --git a/contrib/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/contrib/llvm/include/llvm/Analysis/LoopAnalysisManager.h index 17da516889b0..417ee979ce97 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ b/contrib/llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -37,6 +37,7 @@ #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -58,8 +59,12 @@ struct LoopStandardAnalysisResults { ScalarEvolution &SE; TargetLibraryInfo &TLI; TargetTransformInfo &TTI; + MemorySSA *MSSA; }; +/// Enables memory ssa as a dependency for loop passes. +extern cl::opt<bool> EnableMSSALoopDependency; + /// Extern template declaration for the analysis set for this IR unit. extern template class AllAnalysesOn<Loop>; diff --git a/contrib/llvm/include/llvm/Analysis/LoopInfo.h b/contrib/llvm/include/llvm/Analysis/LoopInfo.h index 70ce9a870517..28afc39727fa 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopInfo.h +++ b/contrib/llvm/include/llvm/Analysis/LoopInfo.h @@ -29,7 +29,7 @@ // in the CFG. There can be strongly connected components in the CFG which // this analysis will not recognize and that will not be represented by a Loop // instance. In particular, a Loop might be inside such a non-loop SCC, or a -// non-loop SCC might contain a sub-SCC which is a Loop. +// non-loop SCC might contain a sub-SCC which is a Loop. // //===----------------------------------------------------------------------===// @@ -46,7 +46,9 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/Allocator.h" #include <algorithm> +#include <utility> namespace llvm { @@ -56,110 +58,145 @@ class Loop; class MDNode; class PHINode; class raw_ostream; -template <class N, bool IsPostDom> -class DominatorTreeBase; -template<class N, class M> class LoopInfoBase; -template<class N, class M> class LoopBase; +template <class N, bool IsPostDom> class DominatorTreeBase; +template <class N, class M> class LoopInfoBase; +template <class N, class M> class LoopBase; //===----------------------------------------------------------------------===// /// Instances of this class are used to represent loops that are detected in the /// flow graph. /// -template<class BlockT, class LoopT> -class LoopBase { +template <class BlockT, class LoopT> class LoopBase { LoopT *ParentLoop; // Loops contained entirely within this one. std::vector<LoopT *> SubLoops; // The list of blocks in this loop. First entry is the header node. - std::vector<BlockT*> Blocks; + std::vector<BlockT *> Blocks; - SmallPtrSet<const BlockT*, 8> DenseBlockSet; + SmallPtrSet<const BlockT *, 8> DenseBlockSet; +#if LLVM_ENABLE_ABI_BREAKING_CHECKS /// Indicator that this loop is no longer a valid loop. bool IsInvalid = false; +#endif LoopBase(const LoopBase<BlockT, LoopT> &) = delete; - const LoopBase<BlockT, LoopT>& - operator=(const LoopBase<BlockT, LoopT> &) = delete; -public: - /// This creates an empty loop. - LoopBase() : ParentLoop(nullptr) {} - ~LoopBase() { - for (size_t i = 0, e = SubLoops.size(); i != e; ++i) - delete SubLoops[i]; - } + const LoopBase<BlockT, LoopT> & + operator=(const LoopBase<BlockT, LoopT> &) = delete; +public: /// Return the nesting level of this loop. An outer-most loop has depth 1, /// for consistency with loop depth values used for basic blocks, where depth /// 0 is used for blocks not inside any loops. unsigned getLoopDepth() const { + assert(!isInvalid() && "Loop not in a valid state!"); unsigned D = 1; for (const LoopT *CurLoop = ParentLoop; CurLoop; CurLoop = CurLoop->ParentLoop) ++D; return D; } - BlockT *getHeader() const { return Blocks.front(); } + BlockT *getHeader() const { return getBlocks().front(); } LoopT *getParentLoop() const { return ParentLoop; } /// This is a raw interface for bypassing addChildLoop. - void setParentLoop(LoopT *L) { ParentLoop = L; } + void setParentLoop(LoopT *L) { + assert(!isInvalid() && "Loop not in a valid state!"); + ParentLoop = L; + } /// Return true if the specified loop is contained within in this loop. bool contains(const LoopT *L) const { - if (L == this) return true; - if (!L) return false; + assert(!isInvalid() && "Loop not in a valid state!"); + if (L == this) + return true; + if (!L) + return false; return contains(L->getParentLoop()); } /// Return true if the specified basic block is in this loop. bool contains(const BlockT *BB) const { + assert(!isInvalid() && "Loop not in a valid state!"); return DenseBlockSet.count(BB); } /// Return true if the specified instruction is in this loop. - template<class InstT> - bool contains(const InstT *Inst) const { + template <class InstT> bool contains(const InstT *Inst) const { return contains(Inst->getParent()); } /// Return the loops contained entirely within this loop. - const std::vector<LoopT *> &getSubLoops() const { return SubLoops; } - std::vector<LoopT *> &getSubLoopsVector() { return SubLoops; } + const std::vector<LoopT *> &getSubLoops() const { + assert(!isInvalid() && "Loop not in a valid state!"); + return SubLoops; + } + std::vector<LoopT *> &getSubLoopsVector() { + assert(!isInvalid() && "Loop not in a valid state!"); + return SubLoops; + } typedef typename std::vector<LoopT *>::const_iterator iterator; - typedef typename std::vector<LoopT *>::const_reverse_iterator - reverse_iterator; - iterator begin() const { return SubLoops.begin(); } - iterator end() const { return SubLoops.end(); } - reverse_iterator rbegin() const { return SubLoops.rbegin(); } - reverse_iterator rend() const { return SubLoops.rend(); } - bool empty() const { return SubLoops.empty(); } + typedef + typename std::vector<LoopT *>::const_reverse_iterator reverse_iterator; + iterator begin() const { return getSubLoops().begin(); } + iterator end() const { return getSubLoops().end(); } + reverse_iterator rbegin() const { return getSubLoops().rbegin(); } + reverse_iterator rend() const { return getSubLoops().rend(); } + bool empty() const { return getSubLoops().empty(); } /// Get a list of the basic blocks which make up this loop. - const std::vector<BlockT*> &getBlocks() const { return Blocks; } - typedef typename std::vector<BlockT*>::const_iterator block_iterator; - block_iterator block_begin() const { return Blocks.begin(); } - block_iterator block_end() const { return Blocks.end(); } + ArrayRef<BlockT *> getBlocks() const { + assert(!isInvalid() && "Loop not in a valid state!"); + return Blocks; + } + typedef typename ArrayRef<BlockT *>::const_iterator block_iterator; + block_iterator block_begin() const { return getBlocks().begin(); } + block_iterator block_end() const { return getBlocks().end(); } inline iterator_range<block_iterator> blocks() const { + assert(!isInvalid() && "Loop not in a valid state!"); return make_range(block_begin(), block_end()); } /// Get the number of blocks in this loop in constant time. + /// Invalidate the loop, indicating that it is no longer a loop. unsigned getNumBlocks() const { + assert(!isInvalid() && "Loop not in a valid state!"); return Blocks.size(); } - /// Invalidate the loop, indicating that it is no longer a loop. - void invalidate() { IsInvalid = true; } - - /// Return true if this loop is no longer valid. - bool isInvalid() { return IsInvalid; } + /// Return a direct, mutable handle to the blocks vector so that we can + /// mutate it efficiently with techniques like `std::remove`. + std::vector<BlockT *> &getBlocksVector() { + assert(!isInvalid() && "Loop not in a valid state!"); + return Blocks; + } + /// Return a direct, mutable handle to the blocks set so that we can + /// mutate it efficiently. + SmallPtrSetImpl<const BlockT *> &getBlocksSet() { + assert(!isInvalid() && "Loop not in a valid state!"); + return DenseBlockSet; + } + + /// Return true if this loop is no longer valid. The only valid use of this + /// helper is "assert(L.isInvalid())" or equivalent, since IsInvalid is set to + /// true by the destructor. In other words, if this accessor returns true, + /// the caller has already triggered UB by calling this accessor; and so it + /// can only be called in a context where a return value of true indicates a + /// programmer error. + bool isInvalid() const { +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + return IsInvalid; +#else + return false; +#endif + } /// True if terminator in the block can branch to another block that is /// outside of the current loop. bool isLoopExiting(const BlockT *BB) const { - for (const auto &Succ : children<const BlockT*>(BB)) { + assert(!isInvalid() && "Loop not in a valid state!"); + for (const auto &Succ : children<const BlockT *>(BB)) { if (!contains(Succ)) return true; } @@ -171,20 +208,22 @@ public: /// This function is useful when there are multiple latches in a loop /// because \fn getLoopLatch will return nullptr in that case. bool isLoopLatch(const BlockT *BB) const { + assert(!isInvalid() && "Loop not in a valid state!"); assert(contains(BB) && "block does not belong to the loop"); BlockT *Header = getHeader(); - auto PredBegin = GraphTraits<Inverse<BlockT*> >::child_begin(Header); - auto PredEnd = GraphTraits<Inverse<BlockT*> >::child_end(Header); + auto PredBegin = GraphTraits<Inverse<BlockT *>>::child_begin(Header); + auto PredEnd = GraphTraits<Inverse<BlockT *>>::child_end(Header); return std::find(PredBegin, PredEnd, BB) != PredEnd; } /// Calculate the number of back edges to the loop header. unsigned getNumBackEdges() const { + assert(!isInvalid() && "Loop not in a valid state!"); unsigned NumBackEdges = 0; BlockT *H = getHeader(); - for (const auto Pred : children<Inverse<BlockT*> >(H)) + for (const auto Pred : children<Inverse<BlockT *>>(H)) if (contains(Pred)) ++NumBackEdges; @@ -210,14 +249,14 @@ public: /// Return all of the successor blocks of this loop. These are the blocks /// _outside of the current loop_ which are branched to. - void getExitBlocks(SmallVectorImpl<BlockT*> &ExitBlocks) const; + void getExitBlocks(SmallVectorImpl<BlockT *> &ExitBlocks) const; /// If getExitBlocks would return exactly one block, return that block. /// Otherwise return null. BlockT *getExitBlock() const; /// Edge type. - typedef std::pair<const BlockT*, const BlockT*> Edge; + typedef std::pair<const BlockT *, const BlockT *> Edge; /// Return all pairs of (_inside_block_,_outside_block_). void getExitEdges(SmallVectorImpl<Edge> &ExitEdges) const; @@ -243,8 +282,9 @@ public: /// Return all loop latch blocks of this loop. A latch block is a block that /// contains a branch back to the header. void getLoopLatches(SmallVectorImpl<BlockT *> &LoopLatches) const { + assert(!isInvalid() && "Loop not in a valid state!"); BlockT *H = getHeader(); - for (const auto Pred : children<Inverse<BlockT*>>(H)) + for (const auto Pred : children<Inverse<BlockT *>>(H)) if (contains(Pred)) LoopLatches.push_back(Pred); } @@ -269,6 +309,7 @@ public: /// Add the specified loop to be a child of this loop. /// This updates the loop depth of the new child. void addChildLoop(LoopT *NewChild) { + assert(!isInvalid() && "Loop not in a valid state!"); assert(!NewChild->ParentLoop && "NewChild already has a parent!"); NewChild->ParentLoop = static_cast<LoopT *>(this); SubLoops.push_back(NewChild); @@ -277,37 +318,49 @@ public: /// This removes the specified child from being a subloop of this loop. The /// loop is not deleted, as it will presumably be inserted into another loop. LoopT *removeChildLoop(iterator I) { + assert(!isInvalid() && "Loop not in a valid state!"); assert(I != SubLoops.end() && "Cannot remove end iterator!"); LoopT *Child = *I; assert(Child->ParentLoop == this && "Child is not a child of this loop!"); - SubLoops.erase(SubLoops.begin()+(I-begin())); + SubLoops.erase(SubLoops.begin() + (I - begin())); Child->ParentLoop = nullptr; return Child; } + /// This removes the specified child from being a subloop of this loop. The + /// loop is not deleted, as it will presumably be inserted into another loop. + LoopT *removeChildLoop(LoopT *Child) { + return removeChildLoop(llvm::find(*this, Child)); + } + /// This adds a basic block directly to the basic block list. /// This should only be used by transformations that create new loops. Other /// transformations should use addBasicBlockToLoop. void addBlockEntry(BlockT *BB) { + assert(!isInvalid() && "Loop not in a valid state!"); Blocks.push_back(BB); DenseBlockSet.insert(BB); } /// interface to reverse Blocks[from, end of loop] in this loop void reverseBlock(unsigned from) { + assert(!isInvalid() && "Loop not in a valid state!"); std::reverse(Blocks.begin() + from, Blocks.end()); } /// interface to do reserve() for Blocks void reserveBlocks(unsigned size) { + assert(!isInvalid() && "Loop not in a valid state!"); Blocks.reserve(size); } /// This method is used to move BB (which must be part of this loop) to be the /// loop header of the loop (the block that dominates all others). void moveToHeader(BlockT *BB) { - if (Blocks[0] == BB) return; - for (unsigned i = 0; ; ++i) { + assert(!isInvalid() && "Loop not in a valid state!"); + if (Blocks[0] == BB) + return; + for (unsigned i = 0;; ++i) { assert(i != Blocks.size() && "Loop does not contain BB!"); if (Blocks[i] == BB) { Blocks[i] = Blocks[0]; @@ -321,6 +374,7 @@ public: /// Blocks as appropriate. This does not update the mapping in the LoopInfo /// class. void removeBlockFromLoop(BlockT *BB) { + assert(!isInvalid() && "Loop not in a valid state!"); auto I = find(Blocks, BB); assert(I != Blocks.end() && "N is not in this list!"); Blocks.erase(I); @@ -332,21 +386,47 @@ public: void verifyLoop() const; /// Verify loop structure of this loop and all nested loops. - void verifyLoopNest(DenseSet<const LoopT*> *Loops) const; + void verifyLoopNest(DenseSet<const LoopT *> *Loops) const; /// Print loop with all the BBs inside it. void print(raw_ostream &OS, unsigned Depth = 0, bool Verbose = false) const; protected: friend class LoopInfoBase<BlockT, LoopT>; + + /// This creates an empty loop. + LoopBase() : ParentLoop(nullptr) {} + explicit LoopBase(BlockT *BB) : ParentLoop(nullptr) { Blocks.push_back(BB); DenseBlockSet.insert(BB); } + + // Since loop passes like SCEV are allowed to key analysis results off of + // `Loop` pointers, we cannot re-use pointers within a loop pass manager. + // This means loop passes should not be `delete` ing `Loop` objects directly + // (and risk a later `Loop` allocation re-using the address of a previous one) + // but should be using LoopInfo::markAsRemoved, which keeps around the `Loop` + // pointer till the end of the lifetime of the `LoopInfo` object. + // + // To make it easier to follow this rule, we mark the destructor as + // non-public. + ~LoopBase() { + for (auto *SubLoop : SubLoops) + SubLoop->~LoopT(); + +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + IsInvalid = true; +#endif + SubLoops.clear(); + Blocks.clear(); + DenseBlockSet.clear(); + ParentLoop = nullptr; + } }; -template<class BlockT, class LoopT> -raw_ostream& operator<<(raw_ostream &OS, const LoopBase<BlockT, LoopT> &Loop) { +template <class BlockT, class LoopT> +raw_ostream &operator<<(raw_ostream &OS, const LoopBase<BlockT, LoopT> &Loop) { Loop.print(OS); return OS; } @@ -354,7 +434,6 @@ raw_ostream& operator<<(raw_ostream &OS, const LoopBase<BlockT, LoopT> &Loop) { // Implementation in LoopInfoImpl.h extern template class LoopBase<BasicBlock, Loop>; - /// Represents a single loop in the control flow graph. Note that not all SCCs /// in the CFG are necessarily loops. class Loop : public LoopBase<BasicBlock, Loop> { @@ -367,21 +446,17 @@ public: public: LocRange() {} LocRange(DebugLoc Start) : Start(std::move(Start)), End(std::move(Start)) {} - LocRange(DebugLoc Start, DebugLoc End) : Start(std::move(Start)), - End(std::move(End)) {} + LocRange(DebugLoc Start, DebugLoc End) + : Start(std::move(Start)), End(std::move(End)) {} const DebugLoc &getStart() const { return Start; } const DebugLoc &getEnd() const { return End; } /// \brief Check for null. /// - explicit operator bool() const { - return Start && End; - } + explicit operator bool() const { return Start && End; } }; - Loop() {} - /// Return true if the specified value is loop invariant. bool isLoopInvariant(const Value *V) const; @@ -464,6 +539,14 @@ public: /// operand should be the node itself. void setLoopID(MDNode *LoopID) const; + /// Add llvm.loop.unroll.disable to this loop's loop id metadata. + /// + /// Remove existing unroll metadata and add unroll disable metadata to + /// indicate the loop has already been unrolled. This prevents a loop + /// from being unrolled more than is directed by a pragma if the loop + /// unrolling pass is run more than once (which it generally is). + void setLoopAlreadyUnrolled(); + /// Return true if no exit block for the loop has a predecessor that is /// outside the loop. bool hasDedicatedExits() const; @@ -499,8 +582,12 @@ public: } private: + Loop() = default; + friend class LoopInfoBase<BasicBlock, Loop>; + friend class LoopBase<BasicBlock, Loop>; explicit Loop(BasicBlock *BB) : LoopBase<BasicBlock, Loop>(BB) {} + ~Loop() = default; }; //===----------------------------------------------------------------------===// @@ -508,25 +595,26 @@ private: /// structures in the specified function. /// -template<class BlockT, class LoopT> -class LoopInfoBase { +template <class BlockT, class LoopT> class LoopInfoBase { // BBMap - Mapping of basic blocks to the inner most loop they occur in DenseMap<const BlockT *, LoopT *> BBMap; std::vector<LoopT *> TopLevelLoops; - std::vector<LoopT *> RemovedLoops; + BumpPtrAllocator LoopAllocator; friend class LoopBase<BlockT, LoopT>; friend class LoopInfo; void operator=(const LoopInfoBase &) = delete; LoopInfoBase(const LoopInfoBase &) = delete; + public: - LoopInfoBase() { } + LoopInfoBase() {} ~LoopInfoBase() { releaseMemory(); } LoopInfoBase(LoopInfoBase &&Arg) : BBMap(std::move(Arg.BBMap)), - TopLevelLoops(std::move(Arg.TopLevelLoops)) { + TopLevelLoops(std::move(Arg.TopLevelLoops)), + LoopAllocator(std::move(Arg.LoopAllocator)) { // We have to clear the arguments top level loops as we've taken ownership. Arg.TopLevelLoops.clear(); } @@ -534,8 +622,10 @@ public: BBMap = std::move(RHS.BBMap); for (auto *L : TopLevelLoops) - delete L; + L->~LoopT(); + TopLevelLoops = std::move(RHS.TopLevelLoops); + LoopAllocator = std::move(RHS.LoopAllocator); RHS.TopLevelLoops.clear(); return *this; } @@ -544,19 +634,22 @@ public: BBMap.clear(); for (auto *L : TopLevelLoops) - delete L; + L->~LoopT(); TopLevelLoops.clear(); - for (auto *L : RemovedLoops) - delete L; - RemovedLoops.clear(); + LoopAllocator.Reset(); + } + + template <typename... ArgsTy> LoopT *AllocateLoop(ArgsTy &&... Args) { + LoopT *Storage = LoopAllocator.Allocate<LoopT>(); + return new (Storage) LoopT(std::forward<ArgsTy>(Args)...); } /// iterator/begin/end - The interface to the top-level loops in the current /// function. /// typedef typename std::vector<LoopT *>::const_iterator iterator; - typedef typename std::vector<LoopT *>::const_reverse_iterator - reverse_iterator; + typedef + typename std::vector<LoopT *>::const_reverse_iterator reverse_iterator; iterator begin() const { return TopLevelLoops.begin(); } iterator end() const { return TopLevelLoops.end(); } reverse_iterator rbegin() const { return TopLevelLoops.rbegin(); } @@ -585,9 +678,7 @@ public: LoopT *getLoopFor(const BlockT *BB) const { return BBMap.lookup(BB); } /// Same as getLoopFor. - const LoopT *operator[](const BlockT *BB) const { - return getLoopFor(BB); - } + const LoopT *operator[](const BlockT *BB) const { return getLoopFor(BB); } /// Return the loop nesting level of the specified block. A depth of 0 means /// the block is not inside any loop. @@ -609,7 +700,7 @@ public: assert(I != end() && "Cannot remove end iterator!"); LoopT *L = *I; assert(!L->getParentLoop() && "Not a top-level loop!"); - TopLevelLoops.erase(TopLevelLoops.begin() + (I-begin())); + TopLevelLoops.erase(TopLevelLoops.begin() + (I - begin())); return L; } @@ -626,8 +717,7 @@ public: /// Replace the specified loop in the top-level loops list with the indicated /// loop. - void changeTopLevelLoop(LoopT *OldLoop, - LoopT *NewLoop) { + void changeTopLevelLoop(LoopT *OldLoop, LoopT *NewLoop) { auto I = find(TopLevelLoops, OldLoop); assert(I != TopLevelLoops.end() && "Old loop not at top level!"); *I = NewLoop; @@ -658,8 +748,10 @@ public: static bool isNotAlreadyContainedIn(const LoopT *SubLoop, const LoopT *ParentLoop) { - if (!SubLoop) return true; - if (SubLoop == ParentLoop) return false; + if (!SubLoop) + return true; + if (SubLoop == ParentLoop) + return false; return isNotAlreadyContainedIn(SubLoop->getParentLoop(), ParentLoop); } @@ -670,6 +762,24 @@ public: void print(raw_ostream &OS) const; void verify(const DominatorTreeBase<BlockT, false> &DomTree) const; + + /// Destroy a loop that has been removed from the `LoopInfo` nest. + /// + /// This runs the destructor of the loop object making it invalid to + /// reference afterward. The memory is retained so that the *pointer* to the + /// loop remains valid. + /// + /// The caller is responsible for removing this loop from the loop nest and + /// otherwise disconnecting it from the broader `LoopInfo` data structures. + /// Callers that don't naturally handle this themselves should probably call + /// `erase' instead. + void destroy(LoopT *L) { + L->~LoopT(); + + // Since LoopAllocator is a BumpPtrAllocator, this Deallocate only poisons + // \c L, but the pointer remains valid for non-dereferencing uses. + LoopAllocator.Deallocate(L); + } }; // Implementation in LoopInfoImpl.h @@ -682,6 +792,7 @@ class LoopInfo : public LoopInfoBase<BasicBlock, Loop> { void operator=(const LoopInfo &) = delete; LoopInfo(const LoopInfo &) = delete; + public: LoopInfo() {} explicit LoopInfo(const DominatorTreeBase<BasicBlock, false> &DomTree); @@ -702,7 +813,7 @@ public: /// the loop forest and parent loops for each block so that \c L is no longer /// referenced, but does not actually delete \c L immediately. The pointer /// will remain valid until this LoopInfo's memory is released. - void markAsRemoved(Loop *L); + void erase(Loop *L); /// Returns true if replacing From with To everywhere is guaranteed to /// preserve LCSSA form. @@ -710,7 +821,8 @@ public: // Preserving LCSSA form is only problematic if the replacing value is an // instruction. Instruction *I = dyn_cast<Instruction>(To); - if (!I) return true; + if (!I) + return true; // If both instructions are defined in the same basic block then replacement // cannot break LCSSA form. if (I->getParent() == From->getParent()) @@ -718,7 +830,8 @@ public: // If the instruction is not defined in a loop then it can safely replace // anything. Loop *ToLoop = getLoopFor(I->getParent()); - if (!ToLoop) return true; + if (!ToLoop) + return true; // If the replacing instruction is defined in the same loop as the original // instruction, or in a loop that contains it as an inner loop, then using // it as a replacement will not break LCSSA form. @@ -798,7 +911,7 @@ public: }; // Allow clients to walk the list of nested loops... -template <> struct GraphTraits<const Loop*> { +template <> struct GraphTraits<const Loop *> { typedef const Loop *NodeRef; typedef LoopInfo::iterator ChildIteratorType; @@ -807,7 +920,7 @@ template <> struct GraphTraits<const Loop*> { static ChildIteratorType child_end(NodeRef N) { return N->end(); } }; -template <> struct GraphTraits<Loop*> { +template <> struct GraphTraits<Loop *> { typedef Loop *NodeRef; typedef LoopInfo::iterator ChildIteratorType; diff --git a/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h b/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h index e9177e68ed77..b3a16b5369f7 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/LoopInfoImpl.h @@ -31,11 +31,12 @@ namespace llvm { /// outside of the loop. These are the blocks _inside of the current loop_ /// which branch out. The returned list is always unique. /// -template<class BlockT, class LoopT> -void LoopBase<BlockT, LoopT>:: -getExitingBlocks(SmallVectorImpl<BlockT *> &ExitingBlocks) const { +template <class BlockT, class LoopT> +void LoopBase<BlockT, LoopT>::getExitingBlocks( + SmallVectorImpl<BlockT *> &ExitingBlocks) const { + assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT*>(BB)) + for (const auto &Succ : children<BlockT *>(BB)) if (!contains(Succ)) { // Not in current loop? It must be an exit block. ExitingBlocks.push_back(BB); @@ -45,9 +46,10 @@ getExitingBlocks(SmallVectorImpl<BlockT *> &ExitingBlocks) const { /// getExitingBlock - If getExitingBlocks would return exactly one block, /// return that block. Otherwise return null. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> BlockT *LoopBase<BlockT, LoopT>::getExitingBlock() const { - SmallVector<BlockT*, 8> ExitingBlocks; + assert(!isInvalid() && "Loop not in a valid state!"); + SmallVector<BlockT *, 8> ExitingBlocks; getExitingBlocks(ExitingBlocks); if (ExitingBlocks.size() == 1) return ExitingBlocks[0]; @@ -57,11 +59,12 @@ BlockT *LoopBase<BlockT, LoopT>::getExitingBlock() const { /// getExitBlocks - Return all of the successor blocks of this loop. These /// are the blocks _outside of the current loop_ which are branched to. /// -template<class BlockT, class LoopT> -void LoopBase<BlockT, LoopT>:: -getExitBlocks(SmallVectorImpl<BlockT*> &ExitBlocks) const { +template <class BlockT, class LoopT> +void LoopBase<BlockT, LoopT>::getExitBlocks( + SmallVectorImpl<BlockT *> &ExitBlocks) const { + assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT*>(BB)) + for (const auto &Succ : children<BlockT *>(BB)) if (!contains(Succ)) // Not in current loop? It must be an exit block. ExitBlocks.push_back(Succ); @@ -69,9 +72,10 @@ getExitBlocks(SmallVectorImpl<BlockT*> &ExitBlocks) const { /// getExitBlock - If getExitBlocks would return exactly one block, /// return that block. Otherwise return null. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> BlockT *LoopBase<BlockT, LoopT>::getExitBlock() const { - SmallVector<BlockT*, 8> ExitBlocks; + assert(!isInvalid() && "Loop not in a valid state!"); + SmallVector<BlockT *, 8> ExitBlocks; getExitBlocks(ExitBlocks); if (ExitBlocks.size() == 1) return ExitBlocks[0]; @@ -79,11 +83,12 @@ BlockT *LoopBase<BlockT, LoopT>::getExitBlock() const { } /// getExitEdges - Return all pairs of (_inside_block_,_outside_block_). -template<class BlockT, class LoopT> -void LoopBase<BlockT, LoopT>:: -getExitEdges(SmallVectorImpl<Edge> &ExitEdges) const { +template <class BlockT, class LoopT> +void LoopBase<BlockT, LoopT>::getExitEdges( + SmallVectorImpl<Edge> &ExitEdges) const { + assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT*>(BB)) + for (const auto &Succ : children<BlockT *>(BB)) if (!contains(Succ)) // Not in current loop? It must be an exit block. ExitEdges.emplace_back(BB, Succ); @@ -97,22 +102,24 @@ getExitEdges(SmallVectorImpl<Edge> &ExitEdges) const { /// /// This method returns null if there is no preheader for the loop. /// -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> BlockT *LoopBase<BlockT, LoopT>::getLoopPreheader() const { + assert(!isInvalid() && "Loop not in a valid state!"); // Keep track of nodes outside the loop branching to the header... BlockT *Out = getLoopPredecessor(); - if (!Out) return nullptr; + if (!Out) + return nullptr; // Make sure we are allowed to hoist instructions into the predecessor. if (!Out->isLegalToHoistInto()) return nullptr; // Make sure there is only one exit out of the preheader. - typedef GraphTraits<BlockT*> BlockTraits; + typedef GraphTraits<BlockT *> BlockTraits; typename BlockTraits::ChildIteratorType SI = BlockTraits::child_begin(Out); ++SI; if (SI != BlockTraits::child_end(Out)) - return nullptr; // Multiple exits from the block, must not be a preheader. + return nullptr; // Multiple exits from the block, must not be a preheader. // The predecessor has exactly one successor, so it is a preheader. return Out; @@ -123,17 +130,18 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopPreheader() const { /// This is less strict that the loop "preheader" concept, which requires /// the predecessor to have exactly one successor. /// -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> BlockT *LoopBase<BlockT, LoopT>::getLoopPredecessor() const { + assert(!isInvalid() && "Loop not in a valid state!"); // Keep track of nodes outside the loop branching to the header... BlockT *Out = nullptr; // Loop over the predecessors of the header node... BlockT *Header = getHeader(); - for (const auto Pred : children<Inverse<BlockT*>>(Header)) { - if (!contains(Pred)) { // If the block is not in the loop... + for (const auto Pred : children<Inverse<BlockT *>>(Header)) { + if (!contains(Pred)) { // If the block is not in the loop... if (Out && Out != Pred) - return nullptr; // Multiple predecessors outside the loop + return nullptr; // Multiple predecessors outside the loop Out = Pred; } } @@ -145,13 +153,15 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopPredecessor() const { /// getLoopLatch - If there is a single latch block for this loop, return it. /// A latch block is a block that contains a branch back to the header. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> BlockT *LoopBase<BlockT, LoopT>::getLoopLatch() const { + assert(!isInvalid() && "Loop not in a valid state!"); BlockT *Header = getHeader(); BlockT *Latch = nullptr; - for (const auto Pred : children<Inverse<BlockT*>>(Header)) { + for (const auto Pred : children<Inverse<BlockT *>>(Header)) { if (contains(Pred)) { - if (Latch) return nullptr; + if (Latch) + return nullptr; Latch = Pred; } } @@ -169,14 +179,15 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopLatch() const { /// to the specified LoopInfo object as being in the current basic block. It /// is not valid to replace the loop header with this method. /// -template<class BlockT, class LoopT> -void LoopBase<BlockT, LoopT>:: -addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase<BlockT, LoopT> &LIB) { +template <class BlockT, class LoopT> +void LoopBase<BlockT, LoopT>::addBasicBlockToLoop( + BlockT *NewBB, LoopInfoBase<BlockT, LoopT> &LIB) { + assert(!isInvalid() && "Loop not in a valid state!"); #ifndef NDEBUG if (!Blocks.empty()) { auto SameHeader = LIB[getHeader()]; - assert(contains(SameHeader) && getHeader() == SameHeader->getHeader() - && "Incorrect LI specified for this loop!"); + assert(contains(SameHeader) && getHeader() == SameHeader->getHeader() && + "Incorrect LI specified for this loop!"); } #endif assert(NewBB && "Cannot add a null basic block to the loop!"); @@ -198,9 +209,10 @@ addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase<BlockT, LoopT> &LIB) { /// the OldChild entry in our children list with NewChild, and updates the /// parent pointer of OldChild to be null and the NewChild to be this loop. /// This updates the loop depth of the new child. -template<class BlockT, class LoopT> -void LoopBase<BlockT, LoopT>:: -replaceChildLoopWith(LoopT *OldChild, LoopT *NewChild) { +template <class BlockT, class LoopT> +void LoopBase<BlockT, LoopT>::replaceChildLoopWith(LoopT *OldChild, + LoopT *NewChild) { + assert(!isInvalid() && "Loop not in a valid state!"); assert(OldChild->ParentLoop == this && "This loop is already broken!"); assert(!NewChild->ParentLoop && "NewChild already has a parent!"); typename std::vector<LoopT *>::iterator I = find(SubLoops, OldChild); @@ -211,46 +223,48 @@ replaceChildLoopWith(LoopT *OldChild, LoopT *NewChild) { } /// verifyLoop - Verify loop structure -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void LoopBase<BlockT, LoopT>::verifyLoop() const { + assert(!isInvalid() && "Loop not in a valid state!"); #ifndef NDEBUG assert(!Blocks.empty() && "Loop header is missing"); // Setup for using a depth-first iterator to visit every block in the loop. - SmallVector<BlockT*, 8> ExitBBs; + SmallVector<BlockT *, 8> ExitBBs; getExitBlocks(ExitBBs); - df_iterator_default_set<BlockT*> VisitSet; + df_iterator_default_set<BlockT *> VisitSet; VisitSet.insert(ExitBBs.begin(), ExitBBs.end()); - df_ext_iterator<BlockT*, df_iterator_default_set<BlockT*>> - BI = df_ext_begin(getHeader(), VisitSet), - BE = df_ext_end(getHeader(), VisitSet); + df_ext_iterator<BlockT *, df_iterator_default_set<BlockT *>> + BI = df_ext_begin(getHeader(), VisitSet), + BE = df_ext_end(getHeader(), VisitSet); // Keep track of the BBs visited. - SmallPtrSet<BlockT*, 8> VisitedBBs; + SmallPtrSet<BlockT *, 8> VisitedBBs; // Check the individual blocks. - for ( ; BI != BE; ++BI) { + for (; BI != BE; ++BI) { BlockT *BB = *BI; - assert(std::any_of(GraphTraits<BlockT*>::child_begin(BB), - GraphTraits<BlockT*>::child_end(BB), - [&](BlockT *B){return contains(B);}) && + assert(std::any_of(GraphTraits<BlockT *>::child_begin(BB), + GraphTraits<BlockT *>::child_end(BB), + [&](BlockT *B) { return contains(B); }) && "Loop block has no in-loop successors!"); - assert(std::any_of(GraphTraits<Inverse<BlockT*> >::child_begin(BB), - GraphTraits<Inverse<BlockT*> >::child_end(BB), - [&](BlockT *B){return contains(B);}) && + assert(std::any_of(GraphTraits<Inverse<BlockT *>>::child_begin(BB), + GraphTraits<Inverse<BlockT *>>::child_end(BB), + [&](BlockT *B) { return contains(B); }) && "Loop block has no in-loop predecessors!"); SmallVector<BlockT *, 2> OutsideLoopPreds; - std::for_each(GraphTraits<Inverse<BlockT*> >::child_begin(BB), - GraphTraits<Inverse<BlockT*> >::child_end(BB), - [&](BlockT *B){if (!contains(B)) + std::for_each(GraphTraits<Inverse<BlockT *>>::child_begin(BB), + GraphTraits<Inverse<BlockT *>>::child_end(BB), + [&](BlockT *B) { + if (!contains(B)) OutsideLoopPreds.push_back(B); }); if (BB == getHeader()) { - assert(!OutsideLoopPreds.empty() && "Loop is unreachable!"); + assert(!OutsideLoopPreds.empty() && "Loop is unreachable!"); } else if (!OutsideLoopPreds.empty()) { // A non-header loop shouldn't be reachable from outside the loop, // though it is permitted if the predecessor is not itself actually @@ -282,8 +296,8 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const { // Each block in each subloop should be contained within this loop. for (block_iterator BI = (*I)->block_begin(), BE = (*I)->block_end(); BI != BE; ++BI) { - assert(contains(*BI) && - "Loop does not contain all the blocks of a subloop!"); + assert(contains(*BI) && + "Loop does not contain all the blocks of a subloop!"); } // Check the parent loop pointer. @@ -295,9 +309,10 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const { } /// verifyLoop - Verify loop structure of this loop and all nested loops. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void LoopBase<BlockT, LoopT>::verifyLoopNest( - DenseSet<const LoopT*> *Loops) const { + DenseSet<const LoopT *> *Loops) const { + assert(!isInvalid() && "Loop not in a valid state!"); Loops->insert(static_cast<const LoopT *>(this)); // Verify this loop. verifyLoop(); @@ -306,30 +321,34 @@ void LoopBase<BlockT, LoopT>::verifyLoopNest( (*I)->verifyLoopNest(Loops); } -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, unsigned Depth, bool Verbose) const { - OS.indent(Depth*2) << "Loop at depth " << getLoopDepth() - << " containing: "; + OS.indent(Depth * 2) << "Loop at depth " << getLoopDepth() << " containing: "; BlockT *H = getHeader(); for (unsigned i = 0; i < getBlocks().size(); ++i) { BlockT *BB = getBlocks()[i]; if (!Verbose) { - if (i) OS << ","; + if (i) + OS << ","; BB->printAsOperand(OS, false); - } else OS << "\n"; - - if (BB == H) OS << "<header>"; - if (isLoopLatch(BB)) OS << "<latch>"; - if (isLoopExiting(BB)) OS << "<exiting>"; + } else + OS << "\n"; + + if (BB == H) + OS << "<header>"; + if (isLoopLatch(BB)) + OS << "<latch>"; + if (isLoopExiting(BB)) + OS << "<exiting>"; if (Verbose) BB->print(OS); } OS << "\n"; for (iterator I = begin(), E = end(); I != E; ++I) - (*I)->print(OS, Depth+2); + (*I)->print(OS, Depth + 2); } //===----------------------------------------------------------------------===// @@ -341,10 +360,10 @@ void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, unsigned Depth, /// this loop are mapped to this loop or a subloop. And all subloops within this /// loop have their parent loop set to this loop or a subloop. template <class BlockT, class LoopT> -static void discoverAndMapSubloop( - LoopT *L, ArrayRef<BlockT *> Backedges, LoopInfoBase<BlockT, LoopT> *LI, - const DomTreeBase<BlockT> &DomTree) { - typedef GraphTraits<Inverse<BlockT*> > InvBlockTraits; +static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT *> Backedges, + LoopInfoBase<BlockT, LoopT> *LI, + const DomTreeBase<BlockT> &DomTree) { + typedef GraphTraits<Inverse<BlockT *>> InvBlockTraits; unsigned NumBlocks = 0; unsigned NumSubloops = 0; @@ -364,13 +383,12 @@ static void discoverAndMapSubloop( LI->changeLoopFor(PredBB, L); ++NumBlocks; if (PredBB == L->getHeader()) - continue; + continue; // Push all block predecessors on the worklist. ReverseCFGWorklist.insert(ReverseCFGWorklist.end(), InvBlockTraits::child_begin(PredBB), InvBlockTraits::child_end(PredBB)); - } - else { + } else { // This is a discovered block. Find its outermost discovered loop. while (LoopT *Parent = Subloop->getParentLoop()) Subloop = Parent; @@ -382,13 +400,13 @@ static void discoverAndMapSubloop( // Discover a subloop of this loop. Subloop->setParentLoop(L); ++NumSubloops; - NumBlocks += Subloop->getBlocks().capacity(); + NumBlocks += Subloop->getBlocksVector().capacity(); PredBB = Subloop->getHeader(); // Continue traversal along predecessors that are not loop-back edges from // within this subloop tree itself. Note that a predecessor may directly // reach another subloop that is not yet discovered to be a subloop of // this loop, which we must traverse. - for (const auto Pred : children<Inverse<BlockT*>>(PredBB)) { + for (const auto Pred : children<Inverse<BlockT *>>(PredBB)) { if (LI->getLoopFor(Pred) != Subloop) ReverseCFGWorklist.push_back(Pred); } @@ -399,15 +417,14 @@ static void discoverAndMapSubloop( } /// Populate all loop data in a stable order during a single forward DFS. -template<class BlockT, class LoopT> -class PopulateLoopsDFS { - typedef GraphTraits<BlockT*> BlockTraits; +template <class BlockT, class LoopT> class PopulateLoopsDFS { + typedef GraphTraits<BlockT *> BlockTraits; typedef typename BlockTraits::ChildIteratorType SuccIterTy; LoopInfoBase<BlockT, LoopT> *LI; + public: - PopulateLoopsDFS(LoopInfoBase<BlockT, LoopT> *li): - LI(li) {} + PopulateLoopsDFS(LoopInfoBase<BlockT, LoopT> *li) : LI(li) {} void traverse(BlockT *EntryBlock); @@ -416,7 +433,7 @@ protected: }; /// Top-level driver for the forward DFS within the loop. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void PopulateLoopsDFS<BlockT, LoopT>::traverse(BlockT *EntryBlock) { for (BlockT *BB : post_order(EntryBlock)) insertIntoLoop(BB); @@ -425,7 +442,7 @@ void PopulateLoopsDFS<BlockT, LoopT>::traverse(BlockT *EntryBlock) { /// Add a single Block to its ancestor loops in PostOrder. If the block is a /// subloop header, add the subloop to its parent in PostOrder, then reverse the /// Block and Subloop vectors of the now complete subloop to achieve RPO. -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) { LoopT *Subloop = LI->getLoopFor(Block); if (Subloop && Block == Subloop->getHeader()) { @@ -463,8 +480,7 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) { /// The Block vectors are inclusive, so step 3 requires loop-depth number of /// insertions per block. template <class BlockT, class LoopT> -void LoopInfoBase<BlockT, LoopT>::analyze( - const DomTreeBase<BlockT> &DomTree) { +void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) { // Postorder traversal of the dominator tree. const DomTreeNodeBase<BlockT> *DomRoot = DomTree.getRootNode(); for (auto DomNode : post_order(DomRoot)) { @@ -473,17 +489,17 @@ void LoopInfoBase<BlockT, LoopT>::analyze( SmallVector<BlockT *, 4> Backedges; // Check each predecessor of the potential loop header. - for (const auto Backedge : children<Inverse<BlockT*>>(Header)) { + for (const auto Backedge : children<Inverse<BlockT *>>(Header)) { // If Header dominates predBB, this is a new loop. Collect the backedges. - if (DomTree.dominates(Header, Backedge) - && DomTree.isReachableFromEntry(Backedge)) { + if (DomTree.dominates(Header, Backedge) && + DomTree.isReachableFromEntry(Backedge)) { Backedges.push_back(Backedge); } } // Perform a backward CFG traversal to discover and map blocks in this loop. if (!Backedges.empty()) { - LoopT *L = new LoopT(Header); - discoverAndMapSubloop(L, ArrayRef<BlockT*>(Backedges), this, DomTree); + LoopT *L = AllocateLoop(Header); + discoverAndMapSubloop(L, ArrayRef<BlockT *>(Backedges), this, DomTree); } } // Perform a single forward CFG traversal to populate block and subloop @@ -542,7 +558,7 @@ LoopInfoBase<BlockT, LoopT>::getLoopsInReverseSiblingPreorder() { } // Debugging -template<class BlockT, class LoopT> +template <class BlockT, class LoopT> void LoopInfoBase<BlockT, LoopT>::print(raw_ostream &OS) const { for (unsigned i = 0; i < TopLevelLoops.size(); ++i) TopLevelLoops[i]->print(OS); @@ -607,13 +623,13 @@ static void compareLoops(const LoopT *L, const LoopT *OtherL, template <class BlockT, class LoopT> void LoopInfoBase<BlockT, LoopT>::verify( const DomTreeBase<BlockT> &DomTree) const { - DenseSet<const LoopT*> Loops; + DenseSet<const LoopT *> Loops; for (iterator I = begin(), E = end(); I != E; ++I) { assert(!(*I)->getParentLoop() && "Top-level loop has a parent!"); (*I)->verifyLoopNest(&Loops); } - // Verify that blocks are mapped to valid loops. +// Verify that blocks are mapped to valid loops. #ifndef NDEBUG for (auto &Entry : BBMap) { const BlockT *BB = Entry.first; diff --git a/contrib/llvm/include/llvm/Analysis/LoopPass.h b/contrib/llvm/include/llvm/Analysis/LoopPass.h index 75e7688bbdc2..86cfecd9df11 100644 --- a/contrib/llvm/include/llvm/Analysis/LoopPass.h +++ b/contrib/llvm/include/llvm/Analysis/LoopPass.h @@ -129,6 +129,9 @@ public: // Add a new loop into the loop queue. void addLoop(Loop &L); + // Mark \p L as deleted. + void markLoopAsDeleted(Loop &L); + //===--------------------------------------------------------------------===// /// SimpleAnalysis - Provides simple interface to update analysis info /// maintained by various passes. Note, if required this interface can @@ -152,6 +155,7 @@ private: std::deque<Loop *> LQ; LoopInfo *LI; Loop *CurrentLoop; + bool CurrentLoopDeleted; }; // This pass is required by the LCSSA transformation. It is used inside diff --git a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h index 23ab372703ee..7d53e34938b7 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -1,4 +1,4 @@ -//===- llvm/Analysis/MemoryBuiltins.h- Calls to memory builtins -*- C++ -*-===// +//==- llvm/Analysis/MemoryBuiltins.h - Calls to memory builtins --*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -15,21 +15,42 @@ #ifndef LLVM_ANALYSIS_MEMORYBUILTINS_H #define LLVM_ANALYSIS_MEMORYBUILTINS_H +#include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/TargetFolder.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/Support/DataTypes.h" +#include <cstdint> +#include <utility> namespace llvm { + +class AllocaInst; +class Argument; class CallInst; -class PointerType; +class ConstantInt; +class ConstantPointerNull; class DataLayout; +class ExtractElementInst; +class ExtractValueInst; +class GEPOperator; +class GlobalAlias; +class GlobalVariable; +class Instruction; +class IntegerType; +class IntrinsicInst; +class IntToPtrInst; +class LLVMContext; +class LoadInst; +class PHINode; +class PointerType; +class SelectInst; class TargetLibraryInfo; class Type; +class UndefValue; class Value; /// \brief Tests if a value is a call or invoke to a library function that @@ -71,8 +92,7 @@ bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, /// is a malloc call. Since CallInst::CreateMalloc() only creates calls, we /// ignore InvokeInst here. const CallInst *extractMallocCall(const Value *I, const TargetLibraryInfo *TLI); -static inline CallInst *extractMallocCall(Value *I, - const TargetLibraryInfo *TLI) { +inline CallInst *extractMallocCall(Value *I, const TargetLibraryInfo *TLI) { return const_cast<CallInst*>(extractMallocCall((const Value*)I, TLI)); } @@ -106,8 +126,7 @@ Value *getMallocArraySize(CallInst *CI, const DataLayout &DL, /// extractCallocCall - Returns the corresponding CallInst if the instruction /// is a calloc call. const CallInst *extractCallocCall(const Value *I, const TargetLibraryInfo *TLI); -static inline CallInst *extractCallocCall(Value *I, - const TargetLibraryInfo *TLI) { +inline CallInst *extractCallocCall(Value *I, const TargetLibraryInfo *TLI) { return const_cast<CallInst*>(extractCallocCall((const Value*)I, TLI)); } @@ -119,11 +138,10 @@ static inline CallInst *extractCallocCall(Value *I, /// isFreeCall - Returns non-null if the value is a call to the builtin free() const CallInst *isFreeCall(const Value *I, const TargetLibraryInfo *TLI); -static inline CallInst *isFreeCall(Value *I, const TargetLibraryInfo *TLI) { +inline CallInst *isFreeCall(Value *I, const TargetLibraryInfo *TLI) { return const_cast<CallInst*>(isFreeCall((const Value*)I, TLI)); } - //===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // @@ -169,13 +187,12 @@ ConstantInt *lowerObjectSizeCall(IntrinsicInst *ObjectSize, const TargetLibraryInfo *TLI, bool MustSucceed); -typedef std::pair<APInt, APInt> SizeOffsetType; +using SizeOffsetType = std::pair<APInt, APInt>; /// \brief Evaluate the size and offset of an object pointed to by a Value* /// statically. Fails if size or offset are not known at compile time. class ObjectSizeOffsetVisitor : public InstVisitor<ObjectSizeOffsetVisitor, SizeOffsetType> { - const DataLayout &DL; const TargetLibraryInfo *TLI; ObjectSizeOpts Options; @@ -229,18 +246,16 @@ private: bool CheckedZextOrTrunc(APInt &I); }; -typedef std::pair<Value*, Value*> SizeOffsetEvalType; - +using SizeOffsetEvalType = std::pair<Value *, Value *>; /// \brief Evaluate the size and offset of an object pointed to by a Value*. /// May create code to compute the result at run-time. class ObjectSizeOffsetEvaluator : public InstVisitor<ObjectSizeOffsetEvaluator, SizeOffsetEvalType> { - - typedef IRBuilder<TargetFolder> BuilderTy; - typedef std::pair<WeakTrackingVH, WeakTrackingVH> WeakEvalType; - typedef DenseMap<const Value*, WeakEvalType> CacheMapTy; - typedef SmallPtrSet<const Value*, 8> PtrSetTy; + using BuilderTy = IRBuilder<TargetFolder>; + using WeakEvalType = std::pair<WeakTrackingVH, WeakTrackingVH>; + using CacheMapTy = DenseMap<const Value *, WeakEvalType>; + using PtrSetTy = SmallPtrSet<const Value *, 8>; const DataLayout &DL; const TargetLibraryInfo *TLI; @@ -255,11 +270,13 @@ class ObjectSizeOffsetEvaluator SizeOffsetEvalType unknown() { return std::make_pair(nullptr, nullptr); } + SizeOffsetEvalType compute_(Value *V); public: ObjectSizeOffsetEvaluator(const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, bool RoundToAlign = false); + SizeOffsetEvalType compute(Value *V); bool knownSize(SizeOffsetEvalType SizeOffset) { @@ -291,6 +308,6 @@ public: SizeOffsetEvalType visitInstruction(Instruction &I); }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_MEMORYBUILTINS_H diff --git a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h index 1dbbf6cc6add..c2974525a6ff 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -1,4 +1,4 @@ -//===- llvm/Analysis/MemoryDependenceAnalysis.h - Memory Deps --*- C++ -*-===// +//===- llvm/Analysis/MemoryDependenceAnalysis.h - Memory Deps ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,26 +15,35 @@ #define LLVM_ANALYSIS_MEMORYDEPENDENCEANALYSIS_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerEmbeddedInt.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/PointerSumType.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/MemoryLocation.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PredIteratorCache.h" -#include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" +#include "llvm/Support/ErrorHandling.h" +#include <cassert> +#include <cstdint> +#include <utility> +#include <vector> namespace llvm { -class Function; -class FunctionPass; -class Instruction; -class CallSite; + class AssumptionCache; -class MemoryDependenceResults; -class PredIteratorCache; +class CallSite; class DominatorTree; +class Function; +class Instruction; +class LoadInst; class PHITransAddr; +class TargetLibraryInfo; +class Value; /// A memory dependence query can return one of three different answers. class MemDepResult { @@ -105,17 +114,17 @@ class MemDepResult { Unknown }; - typedef PointerSumType< + using ValueTy = PointerSumType< DepType, PointerSumTypeMember<Invalid, Instruction *>, PointerSumTypeMember<Clobber, Instruction *>, PointerSumTypeMember<Def, Instruction *>, - PointerSumTypeMember<Other, PointerEmbeddedInt<OtherType, 3>>> - ValueTy; + PointerSumTypeMember<Other, PointerEmbeddedInt<OtherType, 3>>>; ValueTy Value; + explicit MemDepResult(ValueTy V) : Value(V) {} public: - MemDepResult() : Value() {} + MemDepResult() = default; /// get methods: These are static ctor methods for creating various /// MemDepResult kinds. @@ -266,23 +275,23 @@ public: /// internal caching mechanism. class MemoryDependenceResults { // A map from instructions to their dependency. - typedef DenseMap<Instruction *, MemDepResult> LocalDepMapType; + using LocalDepMapType = DenseMap<Instruction *, MemDepResult>; LocalDepMapType LocalDeps; public: - typedef std::vector<NonLocalDepEntry> NonLocalDepInfo; + using NonLocalDepInfo = std::vector<NonLocalDepEntry>; private: /// A pair<Value*, bool> where the bool is true if the dependence is a read /// only dependence, false if read/write. - typedef PointerIntPair<const Value *, 1, bool> ValueIsLoadPair; + using ValueIsLoadPair = PointerIntPair<const Value *, 1, bool>; /// This pair is used when caching information for a block. /// /// If the pointer is null, the cache value is not a full query that starts /// at the specified block. If non-null, the bool indicates whether or not /// the contents of the block was skipped. - typedef PointerIntPair<BasicBlock *, 1, bool> BBSkipFirstBlockPair; + using BBSkipFirstBlockPair = PointerIntPair<BasicBlock *, 1, bool>; /// This record is the information kept for each (value, is load) pair. struct NonLocalPointerInfo { @@ -293,31 +302,32 @@ private: /// The maximum size of the dereferences of the pointer. /// /// May be UnknownSize if the sizes are unknown. - uint64_t Size; + uint64_t Size = MemoryLocation::UnknownSize; /// The AA tags associated with dereferences of the pointer. /// /// The members may be null if there are no tags or conflicting tags. AAMDNodes AATags; - NonLocalPointerInfo() : Size(MemoryLocation::UnknownSize) {} + NonLocalPointerInfo() = default; }; /// Cache storing single nonlocal def for the instruction. /// It is set when nonlocal def would be found in function returning only /// local dependencies. DenseMap<Instruction *, NonLocalDepResult> NonLocalDefsCache; + /// This map stores the cached results of doing a pointer lookup at the /// bottom of a block. /// /// The key of this map is the pointer+isload bit, the value is a list of /// <bb->result> mappings. - typedef DenseMap<ValueIsLoadPair, NonLocalPointerInfo> - CachedNonLocalPointerInfo; + using CachedNonLocalPointerInfo = + DenseMap<ValueIsLoadPair, NonLocalPointerInfo>; CachedNonLocalPointerInfo NonLocalPointerDeps; // A map from instructions to their non-local pointer dependencies. - typedef DenseMap<Instruction *, SmallPtrSet<ValueIsLoadPair, 4>> - ReverseNonLocalPtrDepTy; + using ReverseNonLocalPtrDepTy = + DenseMap<Instruction *, SmallPtrSet<ValueIsLoadPair, 4>>; ReverseNonLocalPtrDepTy ReverseNonLocalPtrDeps; /// This is the instruction we keep for each cached access that we have for @@ -325,17 +335,17 @@ private: /// /// The pointer is an owning pointer and the bool indicates whether we have /// any dirty bits in the set. - typedef std::pair<NonLocalDepInfo, bool> PerInstNLInfo; + using PerInstNLInfo = std::pair<NonLocalDepInfo, bool>; // A map from instructions to their non-local dependencies. - typedef DenseMap<Instruction *, PerInstNLInfo> NonLocalDepMapType; + using NonLocalDepMapType = DenseMap<Instruction *, PerInstNLInfo>; NonLocalDepMapType NonLocalDeps; // A reverse mapping from dependencies to the dependees. This is // used when removing instructions to keep the cache coherent. - typedef DenseMap<Instruction *, SmallPtrSet<Instruction *, 4>> - ReverseDepMapType; + using ReverseDepMapType = + DenseMap<Instruction *, SmallPtrSet<Instruction *, 4>>; ReverseDepMapType ReverseLocalDeps; // A reverse mapping from dependencies to the non-local dependees. @@ -493,10 +503,11 @@ private: class MemoryDependenceAnalysis : public AnalysisInfoMixin<MemoryDependenceAnalysis> { friend AnalysisInfoMixin<MemoryDependenceAnalysis>; + static AnalysisKey Key; public: - typedef MemoryDependenceResults Result; + using Result = MemoryDependenceResults; MemoryDependenceResults run(Function &F, FunctionAnalysisManager &AM); }; @@ -505,10 +516,12 @@ public: /// MemoryDepnedenceResults instance. class MemoryDependenceWrapperPass : public FunctionPass { Optional<MemoryDependenceResults> MemDep; + public: + static char ID; + MemoryDependenceWrapperPass(); ~MemoryDependenceWrapperPass() override; - static char ID; /// Pass Implementation stuff. This doesn't do any analysis eagerly. bool runOnFunction(Function &) override; @@ -522,6 +535,6 @@ public: MemoryDependenceResults &getMemDep() { return *MemDep; } }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_MEMORYDEPENDENCEANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/MemoryLocation.h b/contrib/llvm/include/llvm/Analysis/MemoryLocation.h index f2cb2a123f2e..c1080742e83a 100644 --- a/contrib/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/contrib/llvm/include/llvm/Analysis/MemoryLocation.h @@ -16,6 +16,7 @@ #ifndef LLVM_ANALYSIS_MEMORYLOCATION_H #define LLVM_ANALYSIS_MEMORYLOCATION_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Metadata.h" @@ -68,17 +69,23 @@ public: static MemoryLocation get(const AtomicCmpXchgInst *CXI); static MemoryLocation get(const AtomicRMWInst *RMWI); static MemoryLocation get(const Instruction *Inst) { - if (auto *I = dyn_cast<LoadInst>(Inst)) - return get(I); - else if (auto *I = dyn_cast<StoreInst>(Inst)) - return get(I); - else if (auto *I = dyn_cast<VAArgInst>(Inst)) - return get(I); - else if (auto *I = dyn_cast<AtomicCmpXchgInst>(Inst)) - return get(I); - else if (auto *I = dyn_cast<AtomicRMWInst>(Inst)) - return get(I); - llvm_unreachable("unsupported memory instruction"); + return *MemoryLocation::getOrNone(Inst); + } + static Optional<MemoryLocation> getOrNone(const Instruction *Inst) { + switch (Inst->getOpcode()) { + case Instruction::Load: + return get(cast<LoadInst>(Inst)); + case Instruction::Store: + return get(cast<StoreInst>(Inst)); + case Instruction::VAArg: + return get(cast<VAArgInst>(Inst)); + case Instruction::AtomicCmpXchg: + return get(cast<AtomicCmpXchgInst>(Inst)); + case Instruction::AtomicRMW: + return get(cast<AtomicRMWInst>(Inst)); + default: + return None; + } } /// Return a location representing the source of a memory transfer. diff --git a/contrib/llvm/include/llvm/Analysis/MemorySSA.h b/contrib/llvm/include/llvm/Analysis/MemorySSA.h index 5cec2bfb0cfb..d19f08453ee6 100644 --- a/contrib/llvm/include/llvm/Analysis/MemorySSA.h +++ b/contrib/llvm/include/llvm/Analysis/MemorySSA.h @@ -6,7 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -/// +// /// \file /// \brief This file exposes an interface to building/using memory SSA to /// walk memory instructions using a use/def graph. @@ -67,6 +67,7 @@ /// MemoryDefs are not disambiguated because it would require multiple reaching /// definitions, which would require multiple phis, and multiple memoryaccesses /// per instruction. +// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_MEMORYSSA_H @@ -80,6 +81,7 @@ #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/ADT/simple_ilist.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/PHITransAddr.h" @@ -87,14 +89,12 @@ #include "llvm/IR/DerivedUser.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Module.h" -#include "llvm/IR/OperandTraits.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -107,12 +107,16 @@ namespace llvm { class Function; class Instruction; class MemoryAccess; +class MemorySSAWalker; class LLVMContext; class raw_ostream; + namespace MSSAHelpers { + struct AllAccessTag {}; struct DefsOnlyTag {}; -} + +} // end namespace MSSAHelpers enum { // Used to signify what the default invalid ID is for MemoryAccess's @@ -137,6 +141,11 @@ public: using DefsOnlyType = ilist_node<MemoryAccess, ilist_tag<MSSAHelpers::DefsOnlyTag>>; + MemoryAccess(const MemoryAccess &) = delete; + MemoryAccess &operator=(const MemoryAccess &) = delete; + + void *operator new(size_t) = delete; + // Methods for support type inquiry through isa, cast, and // dyn_cast static bool classof(const Value *V) { @@ -144,19 +153,14 @@ public: return ID == MemoryUseVal || ID == MemoryPhiVal || ID == MemoryDefVal; } - MemoryAccess(const MemoryAccess &) = delete; - MemoryAccess &operator=(const MemoryAccess &) = delete; - - void *operator new(size_t) = delete; - BasicBlock *getBlock() const { return Block; } void print(raw_ostream &OS) const; void dump() const; /// \brief The user iterators for a memory access - typedef user_iterator iterator; - typedef const_user_iterator const_iterator; + using iterator = user_iterator; + using const_iterator = const_user_iterator; /// \brief This iterator walks over all of the defs in a given /// MemoryAccess. For MemoryPhi nodes, this walks arguments. For @@ -194,11 +198,11 @@ public: } protected: - friend class MemorySSA; - friend class MemoryUseOrDef; - friend class MemoryUse; friend class MemoryDef; friend class MemoryPhi; + friend class MemorySSA; + friend class MemoryUse; + friend class MemoryUseOrDef; /// \brief Used by MemorySSA to change the block of a MemoryAccess when it is /// moved. @@ -259,11 +263,13 @@ public: protected: friend class MemorySSA; friend class MemorySSAUpdater; + MemoryUseOrDef(LLVMContext &C, MemoryAccess *DMA, unsigned Vty, DeleteValueTy DeleteValue, Instruction *MI, BasicBlock *BB) : MemoryAccess(C, Vty, DeleteValue, BB, 1), MemoryInst(MI) { setDefiningAccess(DMA); } + void setDefiningAccess(MemoryAccess *DMA, bool Optimized = false) { if (!Optimized) { setOperand(0, DMA); @@ -291,8 +297,7 @@ public: DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); MemoryUse(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB) - : MemoryUseOrDef(C, DMA, MemoryUseVal, deleteMe, MI, BB), - OptimizedID(0) {} + : MemoryUseOrDef(C, DMA, MemoryUseVal, deleteMe, MI, BB) {} // allocate space for exactly one operand void *operator new(size_t s) { return User::operator new(s, 1); } @@ -315,6 +320,7 @@ public: MemoryAccess *getOptimized() const { return getDefiningAccess(); } + void resetOptimized() { OptimizedID = INVALID_MEMORYACCESS_ID; } @@ -325,7 +331,7 @@ protected: private: static void deleteMe(DerivedUser *Self); - unsigned int OptimizedID; + unsigned int OptimizedID = 0; }; template <> @@ -343,12 +349,13 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryUse, MemoryAccess) /// MemoryDef/MemoryPhi. class MemoryDef final : public MemoryUseOrDef { public: + friend class MemorySSA; + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); MemoryDef(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB, unsigned Ver) - : MemoryUseOrDef(C, DMA, MemoryDefVal, deleteMe, MI, BB), - ID(Ver), Optimized(nullptr), OptimizedID(INVALID_MEMORYACCESS_ID) {} + : MemoryUseOrDef(C, DMA, MemoryDefVal, deleteMe, MI, BB), ID(Ver) {} // allocate space for exactly one operand void *operator new(size_t s) { return User::operator new(s, 1); } @@ -361,27 +368,28 @@ public: Optimized = MA; OptimizedID = getDefiningAccess()->getID(); } + MemoryAccess *getOptimized() const { return Optimized; } + bool isOptimized() const { return getOptimized() && getDefiningAccess() && OptimizedID == getDefiningAccess()->getID(); } + void resetOptimized() { OptimizedID = INVALID_MEMORYACCESS_ID; } void print(raw_ostream &OS) const; - friend class MemorySSA; - unsigned getID() const { return ID; } private: static void deleteMe(DerivedUser *Self); const unsigned ID; - MemoryAccess *Optimized; - unsigned int OptimizedID; + MemoryAccess *Optimized = nullptr; + unsigned int OptimizedID = INVALID_MEMORYACCESS_ID; }; template <> @@ -436,8 +444,8 @@ public: // Block iterator interface. This provides access to the list of incoming // basic blocks, which parallels the list of incoming values. - typedef BasicBlock **block_iterator; - typedef BasicBlock *const *const_block_iterator; + using block_iterator = BasicBlock **; + using const_block_iterator = BasicBlock *const *; block_iterator block_begin() { auto *Ref = reinterpret_cast<Use::UserRef *>(op_begin() + ReservedSpace); @@ -477,6 +485,7 @@ public: assert(V && "PHI node got a null value!"); setOperand(I, V); } + static unsigned getOperandNumForIncomingValue(unsigned I) { return I; } static unsigned getIncomingValueNumForOperand(unsigned I) { return I; } @@ -595,12 +604,9 @@ inline void MemoryUseOrDef::resetOptimized() { cast<MemoryUse>(this)->resetOptimized(); } - template <> struct OperandTraits<MemoryPhi> : public HungoffOperandTraits<2> {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryPhi, MemoryAccess) -class MemorySSAWalker; - /// \brief Encapsulates MemorySSA, including all data associated with memory /// accesses. class MemorySSA { @@ -707,11 +713,13 @@ protected: void moveTo(MemoryUseOrDef *What, BasicBlock *BB, AccessList::iterator Where); void moveTo(MemoryUseOrDef *What, BasicBlock *BB, InsertionPlace Point); + // Rename the dominator tree branch rooted at BB. void renamePass(BasicBlock *BB, MemoryAccess *IncomingVal, SmallPtrSetImpl<BasicBlock *> &Visited) { renamePass(DT->getNode(BB), IncomingVal, Visited, true, true); } + void removeFromLookups(MemoryAccess *); void removeFromLists(MemoryAccess *, bool ShouldDelete = true); void insertIntoListsForBlock(MemoryAccess *, const BasicBlock *, @@ -729,6 +737,7 @@ private: void optimizeUses(); void verifyUseInDefs(MemoryAccess *, MemoryAccess *) const; + using AccessMap = DenseMap<const BasicBlock *, std::unique_ptr<AccessList>>; using DefsMap = DenseMap<const BasicBlock *, std::unique_ptr<DefsList>>; @@ -755,6 +764,7 @@ private: // Memory SSA mappings DenseMap<const Value *, MemoryAccess *> ValueToMemoryAccess; + // These two mappings contain the main block to access/def mappings for // MemorySSA. The list contained in PerBlockAccesses really owns all the // MemoryAccesses. @@ -779,8 +789,9 @@ private: // Internal MemorySSA utils, for use by MemorySSA classes and walkers class MemorySSAUtil { protected: - friend class MemorySSAWalker; friend class GVNHoist; + friend class MemorySSAWalker; + // This function should not be used by new passes. static bool defClobbersUseOrDef(MemoryDef *MD, const MemoryUseOrDef *MU, AliasAnalysis &AA); @@ -811,6 +822,7 @@ public: // unique_ptr<MemorySSA> to avoid build breakage on MSVC. struct Result { Result(std::unique_ptr<MemorySSA> &&MSSA) : MSSA(std::move(MSSA)) {} + MemorySSA &getMSSA() { return *MSSA.get(); } std::unique_ptr<MemorySSA> MSSA; @@ -978,6 +990,7 @@ public: assert(MP && "Tried to get phi arg block when not iterating over a PHI"); return MP->getIncomingBlock(ArgNo); } + typename BaseT::iterator::pointer operator*() const { assert(Access && "Tried to access past the end of our iterator"); // Go to the first argument for phis, and the defining access for everything @@ -986,6 +999,7 @@ public: return MP->getIncomingValue(ArgNo); return cast<MemoryUseOrDef>(Access)->getDefiningAccess(); } + using BaseT::operator++; memoryaccess_def_iterator &operator++() { assert(Access && "Hit end of iterator"); diff --git a/contrib/llvm/include/llvm/Analysis/ModuleSummaryAnalysis.h b/contrib/llvm/include/llvm/Analysis/ModuleSummaryAnalysis.h index 4f77170d9f68..9af7859cb4bf 100644 --- a/contrib/llvm/include/llvm/Analysis/ModuleSummaryAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/ModuleSummaryAnalysis.h @@ -14,13 +14,17 @@ #ifndef LLVM_ANALYSIS_MODULESUMMARYANALYSIS_H #define LLVM_ANALYSIS_MODULESUMMARYANALYSIS_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include <functional> namespace llvm { + class BlockFrequencyInfo; +class Function; +class Module; class ProfileSummaryInfo; /// Direct function to compute a \c ModuleSummaryIndex from a given module. @@ -38,10 +42,11 @@ ModuleSummaryIndex buildModuleSummaryIndex( class ModuleSummaryIndexAnalysis : public AnalysisInfoMixin<ModuleSummaryIndexAnalysis> { friend AnalysisInfoMixin<ModuleSummaryIndexAnalysis>; + static AnalysisKey Key; public: - typedef ModuleSummaryIndex Result; + using Result = ModuleSummaryIndex; Result run(Module &M, ModuleAnalysisManager &AM); }; @@ -70,6 +75,7 @@ public: // object for the module, to be written to bitcode or LLVM assembly. // ModulePass *createModuleSummaryIndexWrapperPass(); -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_MODULESUMMARYANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/OptimizationDiagnosticInfo.h b/contrib/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h index 64dd0737a112..26f32acdcda5 100644 --- a/contrib/llvm/include/llvm/Analysis/OptimizationDiagnosticInfo.h +++ b/contrib/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h @@ -1,4 +1,4 @@ -//===- OptimizationDiagnosticInfo.h - Optimization Diagnostic ---*- C++ -*-===// +//===- OptimizationRemarkEmitter.h - Optimization Diagnostic ----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -24,7 +24,6 @@ namespace llvm { class DebugLoc; -class LLVMContext; class Loop; class Pass; class Twine; @@ -69,11 +68,23 @@ public: /// \brief Output the remark via the diagnostic handler and to the /// optimization record file. - /// - /// This is the new interface that should be now used rather than the legacy - /// emit* APIs. void emit(DiagnosticInfoOptimizationBase &OptDiag); + /// \brief Take a lambda that returns a remark which will be emitted. Second + /// argument is only used to restrict this to functions. + template <typename T> + void emit(T RemarkBuilder, decltype(RemarkBuilder()) * = nullptr) { + // Avoid building the remark unless we know there are at least *some* + // remarks enabled. We can't currently check whether remarks are requested + // for the calling pass since that requires actually building the remark. + + if (F->getContext().getDiagnosticsOutputFile() || + F->getContext().getDiagHandlerPtr()->isAnyRemarkEnabled()) { + auto R = RemarkBuilder(); + emit((DiagnosticInfoOptimizationBase &)R); + } + } + /// \brief Whether we allow for extra compile-time budget to perform more /// analysis to produce fewer false positives. /// @@ -81,10 +92,9 @@ public: /// use the extra analysis (1) to filter trivial false positives or (2) to /// provide more context so that non-trivial false positives can be quickly /// detected by the user. - bool allowExtraAnalysis() const { - // For now, only allow this with -fsave-optimization-record since the -Rpass - // options are handled in the front-end. - return F->getContext().getDiagnosticsOutputFile(); + bool allowExtraAnalysis(StringRef PassName) const { + return (F->getContext().getDiagnosticsOutputFile() || + F->getContext().getDiagHandlerPtr()->isAnyRemarkEnabled(PassName)); } private: @@ -154,11 +164,5 @@ public: /// \brief Run the analysis pass over a function and produce BFI. Result run(Function &F, FunctionAnalysisManager &AM); }; - -namespace yaml { -template <> struct MappingTraits<DiagnosticInfoOptimizationBase *> { - static void mapping(IO &io, DiagnosticInfoOptimizationBase *&OptDiag); -}; -} } #endif // LLVM_IR_OPTIMIZATIONDIAGNOSTICINFO_H diff --git a/contrib/llvm/include/llvm/Analysis/PostDominators.h b/contrib/llvm/include/llvm/Analysis/PostDominators.h index 17f2e8eaf4a2..381e65539c4e 100644 --- a/contrib/llvm/include/llvm/Analysis/PostDominators.h +++ b/contrib/llvm/include/llvm/Analysis/PostDominators.h @@ -1,4 +1,4 @@ -//=- llvm/Analysis/PostDominators.h - Post Dominator Calculation-*- C++ -*-===// +//=- llvm/Analysis/PostDominators.h - Post Dominator Calculation --*- C++ -*-=// // // The LLVM Compiler Infrastructure // @@ -14,16 +14,20 @@ #ifndef LLVM_ANALYSIS_POSTDOMINATORS_H #define LLVM_ANALYSIS_POSTDOMINATORS_H +#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" namespace llvm { +class Function; +class raw_ostream; + /// PostDominatorTree Class - Concrete subclass of DominatorTree that is used to /// compute the post-dominator tree. -/// struct PostDominatorTree : public PostDomTreeBase<BasicBlock> { - typedef PostDomTreeBase<BasicBlock> Base; + using Base = PostDomTreeBase<BasicBlock>; /// Handle invalidation explicitly. bool invalidate(Function &F, const PreservedAnalyses &PA, @@ -34,11 +38,12 @@ struct PostDominatorTree : public PostDomTreeBase<BasicBlock> { class PostDominatorTreeAnalysis : public AnalysisInfoMixin<PostDominatorTreeAnalysis> { friend AnalysisInfoMixin<PostDominatorTreeAnalysis>; + static AnalysisKey Key; public: - /// \brief Provide the result typedef for this analysis pass. - typedef PostDominatorTree Result; + /// \brief Provide the result type for this analysis pass. + using Result = PostDominatorTree; /// \brief Run the analysis pass over a function and produce a post dominator /// tree. @@ -52,11 +57,13 @@ class PostDominatorTreePrinterPass public: explicit PostDominatorTreePrinterPass(raw_ostream &OS); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; struct PostDominatorTreeWrapperPass : public FunctionPass { static char ID; // Pass identification, replacement for typeid + PostDominatorTree DT; PostDominatorTreeWrapperPass() : FunctionPass(ID) { @@ -99,6 +106,6 @@ template <> struct GraphTraits<PostDominatorTree*> } }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_POSTDOMINATORS_H diff --git a/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h index 6aaabe1d1889..bd7b00374821 100644 --- a/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/contrib/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -49,6 +49,10 @@ private: void computeThresholds(); // Count thresholds to answer isHotCount and isColdCount queries. Optional<uint64_t> HotCountThreshold, ColdCountThreshold; + // True if the working set size of the code is considered huge, + // because the number of profile counts required to reach the hot + // percentile is above a huge threshold. + Optional<bool> HasHugeWorkingSetSize; public: ProfileSummaryInfo(Module &M) : M(M) {} @@ -84,6 +88,8 @@ public: /// Returns the profile count for \p CallInst. Optional<uint64_t> getProfileCount(const Instruction *CallInst, BlockFrequencyInfo *BFI); + /// Returns true if the working set size of the code is considered huge. + bool hasHugeWorkingSetSize(); /// \brief Returns true if \p F has hot function entry. bool isFunctionEntryHot(const Function *F); /// Returns true if \p F has hot function entry or hot call edge. @@ -104,6 +110,14 @@ public: bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// \brief Returns true if Callsite \p CS is considered cold. bool isColdCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); + /// \brief Returns HotCountThreshold if set. + uint64_t getHotCountThreshold() { + return HotCountThreshold ? HotCountThreshold.getValue() : 0; + } + /// \brief Returns ColdCountThreshold if set. + uint64_t getColdCountThreshold() { + return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; + } }; /// An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo. diff --git a/contrib/llvm/include/llvm/Analysis/PtrUseVisitor.h b/contrib/llvm/include/llvm/Analysis/PtrUseVisitor.h index 2fe7c6725266..9f156a1a6029 100644 --- a/contrib/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/contrib/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -6,6 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides a collection of visitors which walk the (instruction) /// uses of a pointer. These visitors all provide the same essential behavior @@ -16,23 +17,36 @@ /// global variable, or function argument. /// /// FIXME: Provide a variant which doesn't track offsets and is cheaper. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_PTRUSEVISITOR_H #define LLVM_ANALYSIS_PTRUSEVISITOR_H #include "llvm/ADT/APInt.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstVisitor.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/Support/Compiler.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" +#include "llvm/IR/User.h" +#include "llvm/Support/Casting.h" +#include <algorithm> +#include <cassert> +#include <type_traits> namespace llvm { namespace detail { + /// \brief Implementation of non-dependent functionality for \c PtrUseVisitor. /// /// See \c PtrUseVisitor for the public interface and detailed comments about @@ -115,7 +129,8 @@ protected: /// This is used to maintain a worklist fo to-visit uses. This is used to /// make the visit be iterative rather than recursive. struct UseToVisit { - typedef PointerIntPair<Use *, 1, bool> UseAndIsOffsetKnownPair; + using UseAndIsOffsetKnownPair = PointerIntPair<Use *, 1, bool>; + UseAndIsOffsetKnownPair UseAndIsOffsetKnown; APInt Offset; }; @@ -128,7 +143,6 @@ protected: /// @} - /// \name Per-visit state /// This state is reset for each instruction visited. /// @{ @@ -145,7 +159,6 @@ protected: /// @} - /// Note that the constructor is protected because this class must be a base /// class, we can't create instances directly of this class. PtrUseVisitorBase(const DataLayout &DL) : DL(DL) {} @@ -162,6 +175,7 @@ protected: /// offsets and looking through GEPs. bool adjustOffsetForGEP(GetElementPtrInst &GEPI); }; + } // end namespace detail /// \brief A base class for visitors over the uses of a pointer value. @@ -193,7 +207,8 @@ template <typename DerivedT> class PtrUseVisitor : protected InstVisitor<DerivedT>, public detail::PtrUseVisitorBase { friend class InstVisitor<DerivedT>; - typedef InstVisitor<DerivedT> Base; + + using Base = InstVisitor<DerivedT>; public: PtrUseVisitor(const DataLayout &DL) : PtrUseVisitorBase(DL) { @@ -283,6 +298,6 @@ protected: } }; -} +} // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_PTRUSEVISITOR_H diff --git a/contrib/llvm/include/llvm/Analysis/RegionInfo.h b/contrib/llvm/include/llvm/Analysis/RegionInfo.h index 2e34928b28ad..719622359949 100644 --- a/contrib/llvm/include/llvm/Analysis/RegionInfo.h +++ b/contrib/llvm/include/llvm/Analysis/RegionInfo.h @@ -254,7 +254,7 @@ public: template <class Tr> class RegionBase : public RegionNodeBase<Tr> { friend class RegionInfoBase<Tr>; - + using FuncT = typename Tr::FuncT; using BlockT = typename Tr::BlockT; using RegionInfoT = typename Tr::RegionInfoT; @@ -407,6 +407,11 @@ public: /// else NULL. BlockT *getExitingBlock() const; + /// @brief Collect all blocks of this region's single exit edge, if existing. + /// + /// @return True if this region contains all the predecessors of the exit. + bool getExitingBlocks(SmallVectorImpl<BlockT *> &Exitings) const; + /// @brief Is this a simple region? /// /// A region is simple if it has exactly one exit and one entry edge. diff --git a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h index cd4ec0a03a9e..6e522354dd9b 100644 --- a/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -178,6 +178,29 @@ typename RegionBase<Tr>::BlockT *RegionBase<Tr>::getEnteringBlock() const { } template <class Tr> +bool RegionBase<Tr>::getExitingBlocks( + SmallVectorImpl<BlockT *> &Exitings) const { + bool CoverAll = true; + + if (!exit) + return CoverAll; + + for (PredIterTy PI = InvBlockTraits::child_begin(exit), + PE = InvBlockTraits::child_end(exit); + PI != PE; ++PI) { + BlockT *Pred = *PI; + if (contains(Pred)) { + Exitings.push_back(Pred); + continue; + } + + CoverAll = false; + } + + return CoverAll; +} + +template <class Tr> typename RegionBase<Tr>::BlockT *RegionBase<Tr>::getExitingBlock() const { BlockT *exit = getExit(); BlockT *exitingBlock = nullptr; diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h index d1b182755cf8..21b72f3e13c2 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -21,11 +21,21 @@ #ifndef LLVM_ANALYSIS_SCALAREVOLUTION_H #define LLVM_ANALYSIS_SCALAREVOLUTION_H -#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/ConstantRange.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" @@ -33,30 +43,33 @@ #include "llvm/IR/ValueMap.h" #include "llvm/Pass.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/DataTypes.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Compiler.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <memory> +#include <utility> namespace llvm { -class APInt; + class AssumptionCache; +class BasicBlock; class Constant; class ConstantInt; -class DominatorTree; -class Type; -class ScalarEvolution; class DataLayout; -class TargetLibraryInfo; +class DominatorTree; +class GEPOperator; +class Instruction; class LLVMContext; -class Operator; -class SCEV; +class raw_ostream; +class ScalarEvolution; class SCEVAddRecExpr; -class SCEVConstant; -class SCEVExpander; -class SCEVPredicate; class SCEVUnknown; -class Function; - -template <> struct FoldingSetTrait<SCEV>; -template <> struct FoldingSetTrait<SCEVPredicate>; +class StructType; +class TargetLibraryInfo; +class Type; +class Value; /// This class represents an analyzed expression in the program. These are /// opaque objects that the client is not allowed to do much with directly. @@ -74,11 +87,7 @@ class SCEV : public FoldingSetNode { protected: /// This field is initialized to zero and may be used in subclasses to store /// miscellaneous information. - unsigned short SubclassData; - -private: - SCEV(const SCEV &) = delete; - void operator=(const SCEV &) = delete; + unsigned short SubclassData = 0; public: /// NoWrapFlags are bitfield indices into SubclassData. @@ -108,24 +117,22 @@ public: }; explicit SCEV(const FoldingSetNodeIDRef ID, unsigned SCEVTy) - : FastID(ID), SCEVType(SCEVTy), SubclassData(0) {} + : FastID(ID), SCEVType(SCEVTy) {} + SCEV(const SCEV &) = delete; + SCEV &operator=(const SCEV &) = delete; unsigned getSCEVType() const { return SCEVType; } /// Return the LLVM type of this SCEV expression. - /// Type *getType() const; /// Return true if the expression is a constant zero. - /// bool isZero() const; /// Return true if the expression is a constant one. - /// bool isOne() const; /// Return true if the expression is a constant all-ones value. - /// bool isAllOnesValue() const; /// Return true if the specified scev is negated, but not a constant. @@ -136,7 +143,6 @@ public: void print(raw_ostream &OS) const; /// This method is used for debugging. - /// void dump() const; }; @@ -144,10 +150,12 @@ public: // temporary FoldingSetNodeID values. template <> struct FoldingSetTrait<SCEV> : DefaultFoldingSetTrait<SCEV> { static void Profile(const SCEV &X, FoldingSetNodeID &ID) { ID = X.FastID; } + static bool Equals(const SCEV &X, const FoldingSetNodeID &ID, unsigned IDHash, FoldingSetNodeID &TempID) { return ID == X.FastID; } + static unsigned ComputeHash(const SCEV &X, FoldingSetNodeID &TempID) { return X.FastID.ComputeHash(); } @@ -221,7 +229,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const SCEVPredicate &P) { // temporary FoldingSetNodeID values. template <> struct FoldingSetTrait<SCEVPredicate> : DefaultFoldingSetTrait<SCEVPredicate> { - static void Profile(const SCEVPredicate &X, FoldingSetNodeID &ID) { ID = X.FastID; } @@ -230,6 +237,7 @@ struct FoldingSetTrait<SCEVPredicate> : DefaultFoldingSetTrait<SCEVPredicate> { unsigned IDHash, FoldingSetNodeID &TempID) { return ID == X.FastID; } + static unsigned ComputeHash(const SCEVPredicate &X, FoldingSetNodeID &TempID) { return X.FastID.ComputeHash(); @@ -351,6 +359,7 @@ public: /// Returns the set assumed no overflow flags. IncrementWrapFlags getFlags() const { return Flags; } + /// Implementation of the SCEVPredicate interface const SCEV *getExpr() const override; bool implies(const SCEVPredicate *N) const override; @@ -371,11 +380,12 @@ public: /// ScalarEvolution::Preds folding set. This is why the \c add function is sound. class SCEVUnionPredicate final : public SCEVPredicate { private: - typedef DenseMap<const SCEV *, SmallVector<const SCEVPredicate *, 4>> - PredicateMap; + using PredicateMap = + DenseMap<const SCEV *, SmallVector<const SCEVPredicate *, 4>>; /// Vector with references to all predicates in this union. SmallVector<const SCEVPredicate *, 16> Preds; + /// Maps SCEVs to predicates for quick look-ups. PredicateMap SCEVToPreds; @@ -409,6 +419,35 @@ public: } }; +struct ExitLimitQuery { + ExitLimitQuery(const Loop *L, BasicBlock *ExitingBlock, bool AllowPredicates) + : L(L), ExitingBlock(ExitingBlock), AllowPredicates(AllowPredicates) {} + + const Loop *L; + BasicBlock *ExitingBlock; + bool AllowPredicates; +}; + +template <> struct DenseMapInfo<ExitLimitQuery> { + static inline ExitLimitQuery getEmptyKey() { + return ExitLimitQuery(nullptr, nullptr, true); + } + + static inline ExitLimitQuery getTombstoneKey() { + return ExitLimitQuery(nullptr, nullptr, false); + } + + static unsigned getHashValue(ExitLimitQuery Val) { + return hash_combine(hash_combine(Val.L, Val.ExitingBlock), + Val.AllowPredicates); + } + + static bool isEqual(ExitLimitQuery LHS, ExitLimitQuery RHS) { + return LHS.L == RHS.L && LHS.ExitingBlock == RHS.ExitingBlock && + LHS.AllowPredicates == RHS.AllowPredicates; + } +}; + /// The main scalar evolution driver. Because client code (intentionally) /// can't do much with the SCEV objects directly, they must ask this class /// for services. @@ -443,11 +482,542 @@ public: return (SCEV::NoWrapFlags)(Flags & ~OffFlags); } + ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, + DominatorTree &DT, LoopInfo &LI); + ScalarEvolution(ScalarEvolution &&Arg); + ~ScalarEvolution(); + + LLVMContext &getContext() const { return F.getContext(); } + + /// Test if values of the given type are analyzable within the SCEV + /// framework. This primarily includes integer types, and it can optionally + /// include pointer types if the ScalarEvolution class has access to + /// target-specific information. + bool isSCEVable(Type *Ty) const; + + /// Return the size in bits of the specified type, for which isSCEVable must + /// return true. + uint64_t getTypeSizeInBits(Type *Ty) const; + + /// Return a type with the same bitwidth as the given type and which + /// represents how SCEV will treat the given type, for which isSCEVable must + /// return true. For pointer types, this is the pointer-sized integer type. + Type *getEffectiveSCEVType(Type *Ty) const; + + // Returns a wider type among {Ty1, Ty2}. + Type *getWiderType(Type *Ty1, Type *Ty2) const; + + /// Return true if the SCEV is a scAddRecExpr or it contains + /// scAddRecExpr. The result will be cached in HasRecMap. + bool containsAddRecurrence(const SCEV *S); + + /// Erase Value from ValueExprMap and ExprValueMap. + void eraseValueFromMap(Value *V); + + /// Return a SCEV expression for the full generality of the specified + /// expression. + const SCEV *getSCEV(Value *V); + + const SCEV *getConstant(ConstantInt *V); + const SCEV *getConstant(const APInt &Val); + const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); + const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty); + const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); + const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); + const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); + const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); + const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; + return getAddExpr(Ops, Flags, Depth); + } + const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; + return getAddExpr(Ops, Flags, Depth); + } + const SCEV *getMulExpr(SmallVectorImpl<const SCEV *> &Ops, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); + const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; + return getMulExpr(Ops, Flags, Depth); + } + const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0) { + SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; + return getMulExpr(Ops, Flags, Depth); + } + const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUDivExactExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getURemExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, + SCEV::NoWrapFlags Flags); + const SCEV *getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, + const Loop *L, SCEV::NoWrapFlags Flags); + const SCEV *getAddRecExpr(const SmallVectorImpl<const SCEV *> &Operands, + const Loop *L, SCEV::NoWrapFlags Flags) { + SmallVector<const SCEV *, 4> NewOp(Operands.begin(), Operands.end()); + return getAddRecExpr(NewOp, L, Flags); + } + + /// Checks if \p SymbolicPHI can be rewritten as an AddRecExpr under some + /// Predicates. If successful return these <AddRecExpr, Predicates>; + /// The function is intended to be called from PSCEV (the caller will decide + /// whether to actually add the predicates and carry out the rewrites). + Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>> + createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI); + + /// Returns an expression for a GEP + /// + /// \p GEP The GEP. The indices contained in the GEP itself are ignored, + /// instead we use IndexExprs. + /// \p IndexExprs The expressions for the indices. + const SCEV *getGEPExpr(GEPOperator *GEP, + const SmallVectorImpl<const SCEV *> &IndexExprs); + const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getSMaxExpr(SmallVectorImpl<const SCEV *> &Operands); + const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUMaxExpr(SmallVectorImpl<const SCEV *> &Operands); + const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUnknown(Value *V); + const SCEV *getCouldNotCompute(); + + /// Return a SCEV for the constant 0 of a specific type. + const SCEV *getZero(Type *Ty) { return getConstant(Ty, 0); } + + /// Return a SCEV for the constant 1 of a specific type. + const SCEV *getOne(Type *Ty) { return getConstant(Ty, 1); } + + /// Return an expression for sizeof AllocTy that is type IntTy + const SCEV *getSizeOfExpr(Type *IntTy, Type *AllocTy); + + /// Return an expression for offsetof on the given field with type IntTy + const SCEV *getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo); + + /// Return the SCEV object corresponding to -V. + const SCEV *getNegativeSCEV(const SCEV *V, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); + + /// Return the SCEV object corresponding to ~V. + const SCEV *getNotSCEV(const SCEV *V); + + /// Return LHS-RHS. Minus is represented in SCEV as A+B*-1. + const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, + unsigned Depth = 0); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. + const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. + const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. The + /// conversion must not be narrowing. + const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. The + /// conversion must not be narrowing. + const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is extended with + /// unspecified bits. The conversion must not be narrowing. + const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty); + + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. The conversion must not be widening. + const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty); + + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umax operation with them. + const SCEV *getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); + + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umin operation with them. + const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); + + /// Transitively follow the chain of pointer-type operands until reaching a + /// SCEV that does not have a single pointer operand. This returns a + /// SCEVUnknown pointer for well-formed pointer-type expressions, but corner + /// cases do exist. + const SCEV *getPointerBase(const SCEV *V); + + /// Return a SCEV expression for the specified value at the specified scope + /// in the program. The L value specifies a loop nest to evaluate the + /// expression at, where null is the top-level or a specified loop is + /// immediately inside of the loop. + /// + /// This method can be used to compute the exit value for a variable defined + /// in a loop by querying what the value will hold in the parent loop. + /// + /// In the case that a relevant loop exit value cannot be computed, the + /// original value V is returned. + const SCEV *getSCEVAtScope(const SCEV *S, const Loop *L); + + /// This is a convenience function which does getSCEVAtScope(getSCEV(V), L). + const SCEV *getSCEVAtScope(Value *V, const Loop *L); + + /// Test whether entry to the loop is protected by a conditional between LHS + /// and RHS. This is used to help avoid max expressions in loop trip + /// counts, and to eliminate casts. + bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); + + /// Test whether the backedge of the loop is protected by a conditional + /// between LHS and RHS. This is used to to eliminate casts. + bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); + + /// Returns the maximum trip count of the loop if it is a single-exit + /// loop and we can compute a small maximum for that loop. + /// + /// Implemented in terms of the \c getSmallConstantTripCount overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripCount(const Loop *L); + + /// Returns the maximum trip count of this loop as a normal unsigned + /// value. Returns 0 if the trip count is unknown or not constant. This + /// "trip count" assumes that control exits via ExitingBlock. More + /// precisely, it is the number of times that control may reach ExitingBlock + /// before taking the branch. For loops with multiple exits, it may not be + /// the number times that the loop header executes if the loop exits + /// prematurely via another branch. + unsigned getSmallConstantTripCount(const Loop *L, BasicBlock *ExitingBlock); + + /// Returns the upper bound of the loop trip count as a normal unsigned + /// value. + /// Returns 0 if the trip count is unknown or not constant. + unsigned getSmallConstantMaxTripCount(const Loop *L); + + /// Returns the largest constant divisor of the trip count of the + /// loop if it is a single-exit loop and we can compute a small maximum for + /// that loop. + /// + /// Implemented in terms of the \c getSmallConstantTripMultiple overload with + /// the single exiting block passed to it. See that routine for details. + unsigned getSmallConstantTripMultiple(const Loop *L); + + /// Returns the largest constant divisor of the trip count of this loop as a + /// normal unsigned value, if possible. This means that the actual trip + /// count is always a multiple of the returned value (don't forget the trip + /// count could very well be zero as well!). As explained in the comments + /// for getSmallConstantTripCount, this assumes that control exits the loop + /// via ExitingBlock. + unsigned getSmallConstantTripMultiple(const Loop *L, + BasicBlock *ExitingBlock); + + /// Get the expression for the number of loop iterations for which this loop + /// is guaranteed not to exit via ExitingBlock. Otherwise return + /// SCEVCouldNotCompute. + const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); + + /// If the specified loop has a predictable backedge-taken count, return it, + /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count is + /// the number of times the loop header will be branched to from within the + /// loop, assuming there are no abnormal exists like exception throws. This is + /// one less than the trip count of the loop, since it doesn't count the first + /// iteration, when the header is branched to from outside the loop. + /// + /// Note that it is not valid to call this method on a loop without a + /// loop-invariant backedge-taken count (see + /// hasLoopInvariantBackedgeTakenCount). + const SCEV *getBackedgeTakenCount(const Loop *L); + + /// Similar to getBackedgeTakenCount, except it will add a set of + /// SCEV predicates to Predicates that are required to be true in order for + /// the answer to be correct. Predicates can be checked with run-time + /// checks and can be used to perform loop versioning. + const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, + SCEVUnionPredicate &Predicates); + + /// When successful, this returns a SCEVConstant that is greater than or equal + /// to (i.e. a "conservative over-approximation") of the value returend by + /// getBackedgeTakenCount. If such a value cannot be computed, it returns the + /// SCEVCouldNotCompute object. + const SCEV *getMaxBackedgeTakenCount(const Loop *L); + + /// Return true if the backedge taken count is either the value returned by + /// getMaxBackedgeTakenCount or zero. + bool isBackedgeTakenCountMaxOrZero(const Loop *L); + + /// Return true if the specified loop has an analyzable loop-invariant + /// backedge-taken count. + bool hasLoopInvariantBackedgeTakenCount(const Loop *L); + + /// This method should be called by the client when it has changed a loop in + /// a way that may effect ScalarEvolution's ability to compute a trip count, + /// or if the loop is deleted. This call is potentially expensive for large + /// loop bodies. + void forgetLoop(const Loop *L); + + /// This method should be called by the client when it has changed a value + /// in a way that may effect its value, or which may disconnect it from a + /// def-use chain linking it to a loop. + void forgetValue(Value *V); + + /// Called when the client has changed the disposition of values in + /// this loop. + /// + /// We don't have a way to invalidate per-loop dispositions. Clear and + /// recompute is simpler. + void forgetLoopDispositions(const Loop *L) { LoopDispositions.clear(); } + + /// Determine the minimum number of zero bits that S is guaranteed to end in + /// (at every loop iteration). It is, at the same time, the minimum number + /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. + /// If S is guaranteed to be 0, it returns the bitwidth of S. + uint32_t GetMinTrailingZeros(const SCEV *S); + + /// Determine the unsigned range for a particular SCEV. + /// NOTE: This returns a copy of the reference returned by getRangeRef. + ConstantRange getUnsignedRange(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED); + } + + /// Determine the min of the unsigned range for a particular SCEV. + APInt getUnsignedRangeMin(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMin(); + } + + /// Determine the max of the unsigned range for a particular SCEV. + APInt getUnsignedRangeMax(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMax(); + } + + /// Determine the signed range for a particular SCEV. + /// NOTE: This returns a copy of the reference returned by getRangeRef. + ConstantRange getSignedRange(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED); + } + + /// Determine the min of the signed range for a particular SCEV. + APInt getSignedRangeMin(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMin(); + } + + /// Determine the max of the signed range for a particular SCEV. + APInt getSignedRangeMax(const SCEV *S) { + return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMax(); + } + + /// Test if the given expression is known to be negative. + bool isKnownNegative(const SCEV *S); + + /// Test if the given expression is known to be positive. + bool isKnownPositive(const SCEV *S); + + /// Test if the given expression is known to be non-negative. + bool isKnownNonNegative(const SCEV *S); + + /// Test if the given expression is known to be non-positive. + bool isKnownNonPositive(const SCEV *S); + + /// Test if the given expression is known to be non-zero. + bool isKnownNonZero(const SCEV *S); + + /// Test if the given expression is known to satisfy the condition described + /// by Pred, LHS, and RHS. + bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS); + + /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X" + /// is monotonically increasing or decreasing. In the former case set + /// `Increasing` to true and in the latter case set `Increasing` to false. + /// + /// A predicate is said to be monotonically increasing if may go from being + /// false to being true as the loop iterates, but never the other way + /// around. A predicate is said to be monotonically decreasing if may go + /// from being true to being false as the loop iterates, but never the other + /// way around. + bool isMonotonicPredicate(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred, + bool &Increasing); + + /// Return true if the result of the predicate LHS `Pred` RHS is loop + /// invariant with respect to L. Set InvariantPred, InvariantLHS and + /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the + /// loop invariant form of LHS `Pred` RHS. + bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const Loop *L, + ICmpInst::Predicate &InvariantPred, + const SCEV *&InvariantLHS, + const SCEV *&InvariantRHS); + + /// Simplify LHS and RHS in a comparison with predicate Pred. Return true + /// iff any changes were made. If the operands are provably equal or + /// unequal, LHS and RHS are set to the same value and Pred is set to either + /// ICMP_EQ or ICMP_NE. + bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, + const SCEV *&RHS, unsigned Depth = 0); + + /// Return the "disposition" of the given SCEV with respect to the given + /// loop. + LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L); + + /// Return true if the value of the given SCEV is unchanging in the + /// specified loop. + bool isLoopInvariant(const SCEV *S, const Loop *L); + + /// Determine if the SCEV can be evaluated at loop's entry. It is true if it + /// doesn't depend on a SCEVUnknown of an instruction which is dominated by + /// the header of loop L. + bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L); + + /// Return true if the given SCEV changes value in a known way in the + /// specified loop. This property being true implies that the value is + /// variant in the loop AND that we can emit an expression to compute the + /// value of the expression at any particular loop iteration. + bool hasComputableLoopEvolution(const SCEV *S, const Loop *L); + + /// Return the "disposition" of the given SCEV with respect to the given + /// block. + BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB); + + /// Return true if elements that makes up the given SCEV dominate the + /// specified basic block. + bool dominates(const SCEV *S, const BasicBlock *BB); + + /// Return true if elements that makes up the given SCEV properly dominate + /// the specified basic block. + bool properlyDominates(const SCEV *S, const BasicBlock *BB); + + /// Test whether the given SCEV has Op as a direct or indirect operand. + bool hasOperand(const SCEV *S, const SCEV *Op) const; + + /// Return the size of an element read or written by Inst. + const SCEV *getElementSize(Instruction *Inst); + + /// Compute the array dimensions Sizes from the set of Terms extracted from + /// the memory access function of this SCEVAddRecExpr (second step of + /// delinearization). + void findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, + SmallVectorImpl<const SCEV *> &Sizes, + const SCEV *ElementSize); + + void print(raw_ostream &OS) const; + void verify() const; + bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); + + /// Collect parametric terms occurring in step expressions (first step of + /// delinearization). + void collectParametricTerms(const SCEV *Expr, + SmallVectorImpl<const SCEV *> &Terms); + + /// Return in Subscripts the access functions for each dimension in Sizes + /// (third step of delinearization). + void computeAccessFunctions(const SCEV *Expr, + SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<const SCEV *> &Sizes); + + /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the + /// subscripts and sizes of an array access. + /// + /// The delinearization is a 3 step process: the first two steps compute the + /// sizes of each subscript and the third step computes the access functions + /// for the delinearized array: + /// + /// 1. Find the terms in the step functions + /// 2. Compute the array size + /// 3. Compute the access function: divide the SCEV by the array size + /// starting with the innermost dimensions found in step 2. The Quotient + /// is the SCEV to be divided in the next step of the recursion. The + /// Remainder is the subscript of the innermost dimension. Loop over all + /// array dimensions computed in step 2. + /// + /// To compute a uniform array size for several memory accesses to the same + /// object, one can collect in step 1 all the step terms for all the memory + /// accesses, and compute in step 2 a unique array shape. This guarantees + /// that the array shape will be the same across all memory accesses. + /// + /// FIXME: We could derive the result of steps 1 and 2 from a description of + /// the array shape given in metadata. + /// + /// Example: + /// + /// A[][n][m] + /// + /// for i + /// for j + /// for k + /// A[j+k][2i][5i] = + /// + /// The initial SCEV: + /// + /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k] + /// + /// 1. Find the different terms in the step functions: + /// -> [2*m, 5, n*m, n*m] + /// + /// 2. Compute the array size: sort and unique them + /// -> [n*m, 2*m, 5] + /// find the GCD of all the terms = 1 + /// divide by the GCD and erase constant terms + /// -> [n*m, 2*m] + /// GCD = m + /// divide by GCD -> [n, 2] + /// remove constant terms + /// -> [n] + /// size of the array is A[unknown][n][m] + /// + /// 3. Compute the access function + /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m + /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k + /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k + /// The remainder is the subscript of the innermost array dimension: [5i]. + /// + /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n + /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k + /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k + /// The Remainder is the subscript of the next array dimension: [2i]. + /// + /// The subscript of the outermost dimension is the Quotient: [j+k]. + /// + /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i]. + void delinearize(const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts, + SmallVectorImpl<const SCEV *> &Sizes, + const SCEV *ElementSize); + + /// Return the DataLayout associated with the module this SCEV instance is + /// operating on. + const DataLayout &getDataLayout() const { + return F.getParent()->getDataLayout(); + } + + const SCEVPredicate *getEqualPredicate(const SCEV *LHS, const SCEV *RHS); + + const SCEVPredicate * + getWrapPredicate(const SCEVAddRecExpr *AR, + SCEVWrapPredicate::IncrementWrapFlags AddedFlags); + + /// Re-writes the SCEV according to the Predicates in \p A. + const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L, + SCEVUnionPredicate &A); + /// Tries to convert the \p S expression to an AddRec expression, + /// adding additional predicates to \p Preds as required. + const SCEVAddRecExpr *convertSCEVToAddRecWithPredicates( + const SCEV *S, const Loop *L, + SmallPtrSetImpl<const SCEVPredicate *> &Preds); + private: /// A CallbackVH to arrange for ScalarEvolution to be notified whenever a /// Value is deleted. class SCEVCallbackVH final : public CallbackVH { ScalarEvolution *SE; + void deleted() override; void allUsesReplacedWith(Value *New) override; @@ -460,44 +1030,37 @@ private: friend class SCEVUnknown; /// The function we are analyzing. - /// Function &F; /// Does the module have any calls to the llvm.experimental.guard intrinsic /// at all? If this is false, we avoid doing work that will only help if /// thare are guards present in the IR. - /// bool HasGuards; /// The target library information for the target we are targeting. - /// TargetLibraryInfo &TLI; /// The tracker for @llvm.assume intrinsics in this function. AssumptionCache &AC; /// The dominator tree. - /// DominatorTree &DT; /// The loop information for the function we are currently analyzing. - /// LoopInfo &LI; /// This SCEV is used to represent unknown trip counts and things. std::unique_ptr<SCEVCouldNotCompute> CouldNotCompute; - /// The typedef for HasRecMap. - /// - typedef DenseMap<const SCEV *, bool> HasRecMapType; + /// The type for HasRecMap. + using HasRecMapType = DenseMap<const SCEV *, bool>; /// This is a cache to record whether a SCEV contains any scAddRecExpr. HasRecMapType HasRecMap; - /// The typedef for ExprValueMap. - /// - typedef std::pair<Value *, ConstantInt *> ValueOffsetPair; - typedef DenseMap<const SCEV *, SetVector<ValueOffsetPair>> ExprValueMapType; + /// The type for ExprValueMap. + using ValueOffsetPair = std::pair<Value *, ConstantInt *>; + using ExprValueMapType = DenseMap<const SCEV *, SetVector<ValueOffsetPair>>; /// ExprValueMap -- This map records the original values from which /// the SCEV expr is generated from. @@ -521,13 +1084,11 @@ private: /// to V - Offset. ExprValueMapType ExprValueMap; - /// The typedef for ValueExprMap. - /// - typedef DenseMap<SCEVCallbackVH, const SCEV *, DenseMapInfo<Value *>> - ValueExprMapType; + /// The type for ValueExprMap. + using ValueExprMapType = + DenseMap<SCEVCallbackVH, const SCEV *, DenseMapInfo<Value *>>; /// This is a cache of the values we have analyzed so far. - /// ValueExprMapType ValueExprMap; /// Mark predicate values currently being processed by isImpliedCond. @@ -535,15 +1096,18 @@ private: /// Set to true by isLoopBackedgeGuardedByCond when we're walking the set of /// conditions dominating the backedge of a loop. - bool WalkingBEDominatingConds; + bool WalkingBEDominatingConds = false; /// Set to true by isKnownPredicateViaSplitting when we're trying to prove a /// predicate by splitting it into a set of independent predicates. - bool ProvingSplitPredicate; + bool ProvingSplitPredicate = false; /// Memoized values for the GetMinTrailingZeros DenseMap<const SCEV *, uint32_t> MinTrailingZerosCache; + /// Return the Value set from which the SCEV expr is generated. + SetVector<ValueOffsetPair> *getSCEVValues(const SCEV *S); + /// Private helper method for the GetMinTrailingZeros method uint32_t GetMinTrailingZerosImpl(const SCEV *S); @@ -554,7 +1118,9 @@ private: struct ExitLimit { const SCEV *ExactNotTaken; // The exit is not taken exactly this many times const SCEV *MaxNotTaken; // The exit is not taken at most this many times - bool MaxOrZero; // Not taken either exactly MaxNotTaken or zero times + + // Not taken either exactly MaxNotTaken or zero times + bool MaxOrZero = false; /// A set of predicate guards for this ExitLimit. The result is only valid /// if all of the predicates in \c Predicates evaluate to 'true' at @@ -584,6 +1150,8 @@ private: !isa<SCEVCouldNotCompute>(MaxNotTaken); } + bool hasOperand(const SCEV *S) const; + /// Test whether this ExitLimit contains all information. bool hasFullInfo() const { return !isa<SCEVCouldNotCompute>(ExactNotTaken); @@ -596,15 +1164,16 @@ private: PoisoningVH<BasicBlock> ExitingBlock; const SCEV *ExactNotTaken; std::unique_ptr<SCEVUnionPredicate> Predicate; - bool hasAlwaysTruePredicate() const { - return !Predicate || Predicate->isAlwaysTrue(); - } explicit ExitNotTakenInfo(PoisoningVH<BasicBlock> ExitingBlock, const SCEV *ExactNotTaken, std::unique_ptr<SCEVUnionPredicate> Predicate) : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), Predicate(std::move(Predicate)) {} + + bool hasAlwaysTruePredicate() const { + return !Predicate || Predicate->isAlwaysTrue(); + } }; /// Information about the backedge-taken count of a loop. This currently @@ -625,7 +1194,7 @@ private: PointerIntPair<const SCEV *, 1> MaxAndComplete; /// True iff the backedge is taken either exactly Max or zero times. - bool MaxOrZero; + bool MaxOrZero = false; /// \name Helper projection functions on \c MaxAndComplete. /// @{ @@ -634,12 +1203,11 @@ private: /// @} public: - BackedgeTakenInfo() : MaxAndComplete(nullptr, 0), MaxOrZero(false) {} - + BackedgeTakenInfo() : MaxAndComplete(nullptr, 0) {} BackedgeTakenInfo(BackedgeTakenInfo &&) = default; BackedgeTakenInfo &operator=(BackedgeTakenInfo &&) = default; - typedef std::pair<BasicBlock *, ExitLimit> EdgeExitInfo; + using EdgeExitInfo = std::pair<BasicBlock *, ExitLimit>; /// Initialize BackedgeTakenInfo from a list of exact exit counts. BackedgeTakenInfo(SmallVectorImpl<EdgeExitInfo> &&ExitCounts, bool Complete, @@ -826,7 +1394,6 @@ private: /// Implementation code for getSCEVAtScope; called at most once for each /// SCEV+Loop pair. - /// const SCEV *computeSCEVAtScope(const SCEV *S, const Loop *L); /// This looks up computed SCEV values for all instructions that depend on @@ -902,7 +1469,8 @@ private: const ExitLimit &EL); }; - typedef ExitLimitCache ExitLimitCacheTy; + using ExitLimitCacheTy = ExitLimitCache; + ExitLimit computeExitLimitFromCondCached(ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, BasicBlock *TBB, BasicBlock *FBB, @@ -1065,7 +1633,6 @@ private: /// Test if the given expression is known to satisfy the condition described /// by Pred and the known constant ranges of LHS and RHS. - /// bool isKnownPredicateViaConstantRanges(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); @@ -1110,7 +1677,6 @@ private: /// equivalent to proving no signed (resp. unsigned) wrap in /// {`Start`,+,`Step`} if `ExtendOpTy` is `SCEVSignExtendExpr` /// (resp. `SCEVZeroExtendExpr`). - /// template <typename ExtendOpTy> bool proveNoWrapByVaryingStart(const SCEV *Start, const SCEV *Step, const Loop *L); @@ -1150,563 +1716,16 @@ private: /// add recurrence on the loop \p L. bool isAddRecNeverPoison(const Instruction *I, const Loop *L); -public: - ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, - DominatorTree &DT, LoopInfo &LI); - ~ScalarEvolution(); - ScalarEvolution(ScalarEvolution &&Arg); - - LLVMContext &getContext() const { return F.getContext(); } - - /// Test if values of the given type are analyzable within the SCEV - /// framework. This primarily includes integer types, and it can optionally - /// include pointer types if the ScalarEvolution class has access to - /// target-specific information. - bool isSCEVable(Type *Ty) const; - - /// Return the size in bits of the specified type, for which isSCEVable must - /// return true. - uint64_t getTypeSizeInBits(Type *Ty) const; - - /// Return a type with the same bitwidth as the given type and which - /// represents how SCEV will treat the given type, for which isSCEVable must - /// return true. For pointer types, this is the pointer-sized integer type. - Type *getEffectiveSCEVType(Type *Ty) const; - - // Returns a wider type among {Ty1, Ty2}. - Type *getWiderType(Type *Ty1, Type *Ty2) const; - - /// Return true if the SCEV is a scAddRecExpr or it contains - /// scAddRecExpr. The result will be cached in HasRecMap. - /// - bool containsAddRecurrence(const SCEV *S); - - /// Return the Value set from which the SCEV expr is generated. - SetVector<ValueOffsetPair> *getSCEVValues(const SCEV *S); - - /// Erase Value from ValueExprMap and ExprValueMap. - void eraseValueFromMap(Value *V); - - /// Return a SCEV expression for the full generality of the specified - /// expression. - const SCEV *getSCEV(Value *V); - - const SCEV *getConstant(ConstantInt *V); - const SCEV *getConstant(const APInt &Val); - const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); - const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty); - const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); - const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); - const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); - const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); - const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; - return getAddExpr(Ops, Flags, Depth); - } - const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; - return getAddExpr(Ops, Flags, Depth); - } - const SCEV *getMulExpr(SmallVectorImpl<const SCEV *> &Ops, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); - const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; - return getMulExpr(Ops, Flags, Depth); - } - const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0) { - SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; - return getMulExpr(Ops, Flags, Depth); - } - const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUDivExactExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, - SCEV::NoWrapFlags Flags); - const SCEV *getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, - const Loop *L, SCEV::NoWrapFlags Flags); - const SCEV *getAddRecExpr(const SmallVectorImpl<const SCEV *> &Operands, - const Loop *L, SCEV::NoWrapFlags Flags) { - SmallVector<const SCEV *, 4> NewOp(Operands.begin(), Operands.end()); - return getAddRecExpr(NewOp, L, Flags); - } - - /// Checks if \p SymbolicPHI can be rewritten as an AddRecExpr under some - /// Predicates. If successful return these <AddRecExpr, Predicates>; - /// The function is intended to be called from PSCEV (the caller will decide - /// whether to actually add the predicates and carry out the rewrites). - Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>> - createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI); - - /// Returns an expression for a GEP - /// - /// \p GEP The GEP. The indices contained in the GEP itself are ignored, - /// instead we use IndexExprs. - /// \p IndexExprs The expressions for the indices. - const SCEV *getGEPExpr(GEPOperator *GEP, - const SmallVectorImpl<const SCEV *> &IndexExprs); - const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getSMaxExpr(SmallVectorImpl<const SCEV *> &Operands); - const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUMaxExpr(SmallVectorImpl<const SCEV *> &Operands); - const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUnknown(Value *V); - const SCEV *getCouldNotCompute(); - - /// Return a SCEV for the constant 0 of a specific type. - const SCEV *getZero(Type *Ty) { return getConstant(Ty, 0); } - - /// Return a SCEV for the constant 1 of a specific type. - const SCEV *getOne(Type *Ty) { return getConstant(Ty, 1); } - - /// Return an expression for sizeof AllocTy that is type IntTy - /// - const SCEV *getSizeOfExpr(Type *IntTy, Type *AllocTy); - - /// Return an expression for offsetof on the given field with type IntTy - /// - const SCEV *getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo); - - /// Return the SCEV object corresponding to -V. - /// - const SCEV *getNegativeSCEV(const SCEV *V, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); - - /// Return the SCEV object corresponding to ~V. - /// - const SCEV *getNotSCEV(const SCEV *V); - - /// Return LHS-RHS. Minus is represented in SCEV as A+B*-1. - const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, - unsigned Depth = 0); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is zero extended. - const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is sign extended. - const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is zero extended. The - /// conversion must not be narrowing. - const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is sign extended. The - /// conversion must not be narrowing. - const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. If the type must be extended, it is extended with - /// unspecified bits. The conversion must not be narrowing. - const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty); - - /// Return a SCEV corresponding to a conversion of the input value to the - /// specified type. The conversion must not be widening. - const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty); - - /// Promote the operands to the wider of the types using zero-extension, and - /// then perform a umax operation with them. - const SCEV *getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - - /// Promote the operands to the wider of the types using zero-extension, and - /// then perform a umin operation with them. - const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - - /// Transitively follow the chain of pointer-type operands until reaching a - /// SCEV that does not have a single pointer operand. This returns a - /// SCEVUnknown pointer for well-formed pointer-type expressions, but corner - /// cases do exist. - const SCEV *getPointerBase(const SCEV *V); - - /// Return a SCEV expression for the specified value at the specified scope - /// in the program. The L value specifies a loop nest to evaluate the - /// expression at, where null is the top-level or a specified loop is - /// immediately inside of the loop. - /// - /// This method can be used to compute the exit value for a variable defined - /// in a loop by querying what the value will hold in the parent loop. - /// - /// In the case that a relevant loop exit value cannot be computed, the - /// original value V is returned. - const SCEV *getSCEVAtScope(const SCEV *S, const Loop *L); - - /// This is a convenience function which does getSCEVAtScope(getSCEV(V), L). - const SCEV *getSCEVAtScope(Value *V, const Loop *L); - - /// Test whether entry to the loop is protected by a conditional between LHS - /// and RHS. This is used to help avoid max expressions in loop trip - /// counts, and to eliminate casts. - bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); - - /// Test whether the backedge of the loop is protected by a conditional - /// between LHS and RHS. This is used to to eliminate casts. - bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, - const SCEV *LHS, const SCEV *RHS); - - /// Returns the maximum trip count of the loop if it is a single-exit - /// loop and we can compute a small maximum for that loop. - /// - /// Implemented in terms of the \c getSmallConstantTripCount overload with - /// the single exiting block passed to it. See that routine for details. - unsigned getSmallConstantTripCount(const Loop *L); - - /// Returns the maximum trip count of this loop as a normal unsigned - /// value. Returns 0 if the trip count is unknown or not constant. This - /// "trip count" assumes that control exits via ExitingBlock. More - /// precisely, it is the number of times that control may reach ExitingBlock - /// before taking the branch. For loops with multiple exits, it may not be - /// the number times that the loop header executes if the loop exits - /// prematurely via another branch. - unsigned getSmallConstantTripCount(const Loop *L, BasicBlock *ExitingBlock); - - /// Returns the upper bound of the loop trip count as a normal unsigned - /// value. - /// Returns 0 if the trip count is unknown or not constant. - unsigned getSmallConstantMaxTripCount(const Loop *L); - - /// Returns the largest constant divisor of the trip count of the - /// loop if it is a single-exit loop and we can compute a small maximum for - /// that loop. - /// - /// Implemented in terms of the \c getSmallConstantTripMultiple overload with - /// the single exiting block passed to it. See that routine for details. - unsigned getSmallConstantTripMultiple(const Loop *L); - - /// Returns the largest constant divisor of the trip count of this loop as a - /// normal unsigned value, if possible. This means that the actual trip - /// count is always a multiple of the returned value (don't forget the trip - /// count could very well be zero as well!). As explained in the comments - /// for getSmallConstantTripCount, this assumes that control exits the loop - /// via ExitingBlock. - unsigned getSmallConstantTripMultiple(const Loop *L, - BasicBlock *ExitingBlock); - - /// Get the expression for the number of loop iterations for which this loop - /// is guaranteed not to exit via ExitingBlock. Otherwise return - /// SCEVCouldNotCompute. - const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); - - /// If the specified loop has a predictable backedge-taken count, return it, - /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count is - /// the number of times the loop header will be branched to from within the - /// loop, assuming there are no abnormal exists like exception throws. This is - /// one less than the trip count of the loop, since it doesn't count the first - /// iteration, when the header is branched to from outside the loop. - /// - /// Note that it is not valid to call this method on a loop without a - /// loop-invariant backedge-taken count (see - /// hasLoopInvariantBackedgeTakenCount). - /// - const SCEV *getBackedgeTakenCount(const Loop *L); - - /// Similar to getBackedgeTakenCount, except it will add a set of - /// SCEV predicates to Predicates that are required to be true in order for - /// the answer to be correct. Predicates can be checked with run-time - /// checks and can be used to perform loop versioning. - const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Predicates); - - /// When successful, this returns a SCEVConstant that is greater than or equal - /// to (i.e. a "conservative over-approximation") of the value returend by - /// getBackedgeTakenCount. If such a value cannot be computed, it returns the - /// SCEVCouldNotCompute object. - const SCEV *getMaxBackedgeTakenCount(const Loop *L); - - /// Return true if the backedge taken count is either the value returned by - /// getMaxBackedgeTakenCount or zero. - bool isBackedgeTakenCountMaxOrZero(const Loop *L); - - /// Return true if the specified loop has an analyzable loop-invariant - /// backedge-taken count. - bool hasLoopInvariantBackedgeTakenCount(const Loop *L); - - /// This method should be called by the client when it has changed a loop in - /// a way that may effect ScalarEvolution's ability to compute a trip count, - /// or if the loop is deleted. This call is potentially expensive for large - /// loop bodies. - void forgetLoop(const Loop *L); - - /// This method should be called by the client when it has changed a value - /// in a way that may effect its value, or which may disconnect it from a - /// def-use chain linking it to a loop. - void forgetValue(Value *V); - - /// Called when the client has changed the disposition of values in - /// this loop. - /// - /// We don't have a way to invalidate per-loop dispositions. Clear and - /// recompute is simpler. - void forgetLoopDispositions(const Loop *L) { LoopDispositions.clear(); } - - /// Determine the minimum number of zero bits that S is guaranteed to end in - /// (at every loop iteration). It is, at the same time, the minimum number - /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. - /// If S is guaranteed to be 0, it returns the bitwidth of S. - uint32_t GetMinTrailingZeros(const SCEV *S); - - /// Determine the unsigned range for a particular SCEV. - /// NOTE: This returns a copy of the reference returned by getRangeRef. - ConstantRange getUnsignedRange(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED); - } - - /// Determine the min of the unsigned range for a particular SCEV. - APInt getUnsignedRangeMin(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMin(); - } - - /// Determine the max of the unsigned range for a particular SCEV. - APInt getUnsignedRangeMax(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_UNSIGNED).getUnsignedMax(); - } - - /// Determine the signed range for a particular SCEV. - /// NOTE: This returns a copy of the reference returned by getRangeRef. - ConstantRange getSignedRange(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED); - } - - /// Determine the min of the signed range for a particular SCEV. - APInt getSignedRangeMin(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMin(); - } - - /// Determine the max of the signed range for a particular SCEV. - APInt getSignedRangeMax(const SCEV *S) { - return getRangeRef(S, HINT_RANGE_SIGNED).getSignedMax(); - } - - /// Test if the given expression is known to be negative. - /// - bool isKnownNegative(const SCEV *S); - - /// Test if the given expression is known to be positive. - /// - bool isKnownPositive(const SCEV *S); - - /// Test if the given expression is known to be non-negative. - /// - bool isKnownNonNegative(const SCEV *S); - - /// Test if the given expression is known to be non-positive. - /// - bool isKnownNonPositive(const SCEV *S); - - /// Test if the given expression is known to be non-zero. - /// - bool isKnownNonZero(const SCEV *S); - - /// Test if the given expression is known to satisfy the condition described - /// by Pred, LHS, and RHS. - /// - bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS); - - /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X" - /// is monotonically increasing or decreasing. In the former case set - /// `Increasing` to true and in the latter case set `Increasing` to false. - /// - /// A predicate is said to be monotonically increasing if may go from being - /// false to being true as the loop iterates, but never the other way - /// around. A predicate is said to be monotonically decreasing if may go - /// from being true to being false as the loop iterates, but never the other - /// way around. - bool isMonotonicPredicate(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred, - bool &Increasing); - - /// Return true if the result of the predicate LHS `Pred` RHS is loop - /// invariant with respect to L. Set InvariantPred, InvariantLHS and - /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the - /// loop invariant form of LHS `Pred` RHS. - bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, - const SCEV *RHS, const Loop *L, - ICmpInst::Predicate &InvariantPred, - const SCEV *&InvariantLHS, - const SCEV *&InvariantRHS); - - /// Simplify LHS and RHS in a comparison with predicate Pred. Return true - /// iff any changes were made. If the operands are provably equal or - /// unequal, LHS and RHS are set to the same value and Pred is set to either - /// ICMP_EQ or ICMP_NE. - /// - bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, - const SCEV *&RHS, unsigned Depth = 0); - - /// Return the "disposition" of the given SCEV with respect to the given - /// loop. - LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L); - - /// Return true if the value of the given SCEV is unchanging in the - /// specified loop. - bool isLoopInvariant(const SCEV *S, const Loop *L); - - /// Determine if the SCEV can be evaluated at loop's entry. It is true if it - /// doesn't depend on a SCEVUnknown of an instruction which is dominated by - /// the header of loop L. - bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L); - - /// Return true if the given SCEV changes value in a known way in the - /// specified loop. This property being true implies that the value is - /// variant in the loop AND that we can emit an expression to compute the - /// value of the expression at any particular loop iteration. - bool hasComputableLoopEvolution(const SCEV *S, const Loop *L); - - /// Return the "disposition" of the given SCEV with respect to the given - /// block. - BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB); - - /// Return true if elements that makes up the given SCEV dominate the - /// specified basic block. - bool dominates(const SCEV *S, const BasicBlock *BB); - - /// Return true if elements that makes up the given SCEV properly dominate - /// the specified basic block. - bool properlyDominates(const SCEV *S, const BasicBlock *BB); - - /// Test whether the given SCEV has Op as a direct or indirect operand. - bool hasOperand(const SCEV *S, const SCEV *Op) const; - - /// Return the size of an element read or written by Inst. - const SCEV *getElementSize(Instruction *Inst); - - /// Compute the array dimensions Sizes from the set of Terms extracted from - /// the memory access function of this SCEVAddRecExpr (second step of - /// delinearization). - void findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, - SmallVectorImpl<const SCEV *> &Sizes, - const SCEV *ElementSize); - - void print(raw_ostream &OS) const; - void verify() const; - bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &Inv); - - /// Collect parametric terms occurring in step expressions (first step of - /// delinearization). - void collectParametricTerms(const SCEV *Expr, - SmallVectorImpl<const SCEV *> &Terms); - - /// Return in Subscripts the access functions for each dimension in Sizes - /// (third step of delinearization). - void computeAccessFunctions(const SCEV *Expr, - SmallVectorImpl<const SCEV *> &Subscripts, - SmallVectorImpl<const SCEV *> &Sizes); - - /// Split this SCEVAddRecExpr into two vectors of SCEVs representing the - /// subscripts and sizes of an array access. - /// - /// The delinearization is a 3 step process: the first two steps compute the - /// sizes of each subscript and the third step computes the access functions - /// for the delinearized array: - /// - /// 1. Find the terms in the step functions - /// 2. Compute the array size - /// 3. Compute the access function: divide the SCEV by the array size - /// starting with the innermost dimensions found in step 2. The Quotient - /// is the SCEV to be divided in the next step of the recursion. The - /// Remainder is the subscript of the innermost dimension. Loop over all - /// array dimensions computed in step 2. - /// - /// To compute a uniform array size for several memory accesses to the same - /// object, one can collect in step 1 all the step terms for all the memory - /// accesses, and compute in step 2 a unique array shape. This guarantees - /// that the array shape will be the same across all memory accesses. - /// - /// FIXME: We could derive the result of steps 1 and 2 from a description of - /// the array shape given in metadata. - /// - /// Example: - /// - /// A[][n][m] - /// - /// for i - /// for j - /// for k - /// A[j+k][2i][5i] = - /// - /// The initial SCEV: - /// - /// A[{{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k] - /// - /// 1. Find the different terms in the step functions: - /// -> [2*m, 5, n*m, n*m] - /// - /// 2. Compute the array size: sort and unique them - /// -> [n*m, 2*m, 5] - /// find the GCD of all the terms = 1 - /// divide by the GCD and erase constant terms - /// -> [n*m, 2*m] - /// GCD = m - /// divide by GCD -> [n, 2] - /// remove constant terms - /// -> [n] - /// size of the array is A[unknown][n][m] - /// - /// 3. Compute the access function - /// a. Divide {{{0,+,2*m+5}_i, +, n*m}_j, +, n*m}_k by the innermost size m - /// Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k - /// Remainder: {{{0,+,5}_i, +, 0}_j, +, 0}_k - /// The remainder is the subscript of the innermost array dimension: [5i]. - /// - /// b. Divide Quotient: {{{0,+,2}_i, +, n}_j, +, n}_k by next outer size n - /// Quotient: {{{0,+,0}_i, +, 1}_j, +, 1}_k - /// Remainder: {{{0,+,2}_i, +, 0}_j, +, 0}_k - /// The Remainder is the subscript of the next array dimension: [2i]. - /// - /// The subscript of the outermost dimension is the Quotient: [j+k]. - /// - /// Overall, we have: A[][n][m], and the access function: A[j+k][2i][5i]. - void delinearize(const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts, - SmallVectorImpl<const SCEV *> &Sizes, - const SCEV *ElementSize); - - /// Return the DataLayout associated with the module this SCEV instance is - /// operating on. - const DataLayout &getDataLayout() const { - return F.getParent()->getDataLayout(); - } - - const SCEVPredicate *getEqualPredicate(const SCEV *LHS, const SCEV *RHS); - - const SCEVPredicate * - getWrapPredicate(const SCEVAddRecExpr *AR, - SCEVWrapPredicate::IncrementWrapFlags AddedFlags); - - /// Re-writes the SCEV according to the Predicates in \p A. - const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L, - SCEVUnionPredicate &A); - /// Tries to convert the \p S expression to an AddRec expression, - /// adding additional predicates to \p Preds as required. - const SCEVAddRecExpr *convertSCEVToAddRecWithPredicates( - const SCEV *S, const Loop *L, - SmallPtrSetImpl<const SCEVPredicate *> &Preds); - -private: - /// Similar to createAddRecFromPHI, but with the additional flexibility of + /// Similar to createAddRecFromPHI, but with the additional flexibility of /// suggesting runtime overflow checks in case casts are encountered. /// If successful, the analysis records that for this loop, \p SymbolicPHI, /// which is the UnknownSCEV currently representing the PHI, can be rewritten /// into an AddRec, assuming some predicates; The function then returns the /// AddRec and the predicates as a pair, and caches this pair in /// PredicatedSCEVRewrites. - /// If the analysis is not successful, a mapping from the \p SymbolicPHI to + /// If the analysis is not successful, a mapping from the \p SymbolicPHI to /// itself (with no predicates) is recorded, and a nullptr with an empty - /// predicates vector is returned as a pair. + /// predicates vector is returned as a pair. Optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>> createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI); @@ -1715,6 +1734,16 @@ private: const SCEV *computeBECount(const SCEV *Delta, const SCEV *Stride, bool Equality); + /// Compute the maximum backedge count based on the range of values + /// permitted by Start, End, and Stride. This is for loops of the form + /// {Start, +, Stride} LT End. + /// + /// Precondition: the induction variable is known to be positive. We *don't* + /// assert these preconditions so please be careful. + const SCEV *computeMaxBECountForLT(const SCEV *Start, const SCEV *Stride, + const SCEV *End, unsigned BitWidth, + bool IsSigned); + /// Verify if an linear IV with positive stride can overflow when in a /// less-than comparison, knowing the invariant term of the comparison, /// the stride and the knowledge of NSW/NUW flags on the recurrence. @@ -1735,31 +1764,39 @@ private: const SCEV *getOrCreateMulExpr(SmallVectorImpl<const SCEV *> &Ops, SCEV::NoWrapFlags Flags); -private: + /// Find all of the loops transitively used in \p S, and update \c LoopUsers + /// accordingly. + void addToLoopUseLists(const SCEV *S); + FoldingSet<SCEV> UniqueSCEVs; FoldingSet<SCEVPredicate> UniquePreds; BumpPtrAllocator SCEVAllocator; + /// This maps loops to a list of SCEV expressions that (transitively) use said + /// loop. + DenseMap<const Loop *, SmallVector<const SCEV *, 4>> LoopUsers; + /// Cache tentative mappings from UnknownSCEVs in a Loop, to a SCEV expression /// they can be rewritten into under certain predicates. DenseMap<std::pair<const SCEVUnknown *, const Loop *>, std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>> PredicatedSCEVRewrites; - + /// The head of a linked list of all SCEVUnknown values that have been /// allocated. This is used by releaseMemory to locate them all and call /// their destructors. - SCEVUnknown *FirstUnknown; + SCEVUnknown *FirstUnknown = nullptr; }; /// Analysis pass that exposes the \c ScalarEvolution for a function. class ScalarEvolutionAnalysis : public AnalysisInfoMixin<ScalarEvolutionAnalysis> { friend AnalysisInfoMixin<ScalarEvolutionAnalysis>; + static AnalysisKey Key; public: - typedef ScalarEvolution Result; + using Result = ScalarEvolution; ScalarEvolution run(Function &F, FunctionAnalysisManager &AM); }; @@ -1771,6 +1808,7 @@ class ScalarEvolutionPrinterPass public: explicit ScalarEvolutionPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; @@ -1808,6 +1846,7 @@ public: class PredicatedScalarEvolution { public: PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L); + const SCEVUnionPredicate &getUnionPredicate() const; /// Returns the SCEV expression of V, in the context of the current SCEV @@ -1845,6 +1884,11 @@ public: /// The printed text is indented by \p Depth. void print(raw_ostream &OS, unsigned Depth) const; + /// Check if \p AR1 and \p AR2 are equal, while taking into account + /// Equal predicates in Preds. + bool areAddRecsEqualWithPreds(const SCEVAddRecExpr *AR1, + const SCEVAddRecExpr *AR2) const; + private: /// Increments the version number of the predicate. This needs to be called /// every time the SCEV predicate changes. @@ -1852,7 +1896,7 @@ private: /// Holds a SCEV and the version number of the SCEV predicate used to /// perform the rewrite of the expression. - typedef std::pair<unsigned, const SCEV *> RewriteEntry; + using RewriteEntry = std::pair<unsigned, const SCEV *>; /// Maps a SCEV to the rewrite result of that SCEV at a certain version /// number. If this number doesn't match the current Generation, we will @@ -1878,11 +1922,12 @@ private: /// expression we mark it with the version of the predicate. We use this to /// figure out if the predicate has changed from the last rewrite of the /// SCEV. If so, we need to perform a new rewrite. - unsigned Generation; + unsigned Generation = 0; /// The backedge taken count. - const SCEV *BackedgeCount; + const SCEV *BackedgeCount = nullptr; }; -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_SCALAREVOLUTION_H diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h index 7d16f34e54cb..4578e0da8ab2 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -27,9 +27,16 @@ namespace llvm { class TargetTransformInfo; /// Return true if the given expression is safe to expand in the sense that - /// all materialized values are safe to speculate. + /// all materialized values are safe to speculate anywhere their operands are + /// defined. bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE); + /// Return true if the given expression is safe to expand in the sense that + /// all materialized values are defined and safe to speculate at the specified + /// location and their operands are defined at this location. + bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint, + ScalarEvolution &SE); + /// This class uses information about analyze scalars to rewrite expressions /// in canonical form. /// diff --git a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index 56ddb5028d6d..acf83455cdcd 100644 --- a/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/contrib/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -14,15 +14,27 @@ #ifndef LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H #define LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" +#include <cassert> +#include <cstddef> namespace llvm { - class ConstantInt; - class ConstantRange; - class DominatorTree; + +class APInt; +class Constant; +class ConstantRange; +class Loop; +class Type; enum SCEVTypes { // These should be ordered in terms of increasing complexity to make the @@ -37,8 +49,10 @@ namespace llvm { friend class ScalarEvolution; ConstantInt *V; + SCEVConstant(const FoldingSetNodeIDRef ID, ConstantInt *v) : SCEV(ID, scConstant), V(v) {} + public: ConstantInt *getValue() const { return V; } const APInt &getAPInt() const { return getValue()->getValue(); } @@ -117,7 +131,6 @@ namespace llvm { } }; - /// This node is a base class providing common functionality for /// n'ary operators. class SCEVNAryExpr : public SCEV { @@ -135,13 +148,15 @@ namespace llvm { public: size_t getNumOperands() const { return NumOperands; } + const SCEV *getOperand(unsigned i) const { assert(i < NumOperands && "Operand index out of range!"); return Operands[i]; } - typedef const SCEV *const *op_iterator; - typedef iterator_range<op_iterator> op_range; + using op_iterator = const SCEV *const *; + using op_range = iterator_range<op_iterator>; + op_iterator op_begin() const { return Operands; } op_iterator op_end() const { return Operands + NumOperands; } op_range operands() const { @@ -198,15 +213,13 @@ namespace llvm { } }; - /// This node represents an addition of some number of SCEVs. class SCEVAddExpr : public SCEVCommutativeExpr { friend class ScalarEvolution; SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, scAddExpr, O, N) { - } + : SCEVCommutativeExpr(ID, scAddExpr, O, N) {} public: Type *getType() const { @@ -222,15 +235,13 @@ namespace llvm { } }; - /// This node represents multiplication of some number of SCEVs. class SCEVMulExpr : public SCEVCommutativeExpr { friend class ScalarEvolution; SCEVMulExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, scMulExpr, O, N) { - } + : SCEVCommutativeExpr(ID, scMulExpr, O, N) {} public: /// Methods for support type inquiry through isa, cast, and dyn_cast: @@ -239,13 +250,13 @@ namespace llvm { } }; - /// This class represents a binary unsigned division operation. class SCEVUDivExpr : public SCEV { friend class ScalarEvolution; const SCEV *LHS; const SCEV *RHS; + SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs) : SCEV(ID, scUDivExpr), LHS(lhs), RHS(rhs) {} @@ -268,7 +279,6 @@ namespace llvm { } }; - /// This node represents a polynomial recurrence on the trip count /// of the specified loop. This is the primary focus of the /// ScalarEvolution framework; all the other SCEV subclasses are @@ -368,7 +378,6 @@ namespace llvm { } }; - /// This class represents an unsigned maximum selection. class SCEVUMaxExpr : public SCEVCommutativeExpr { friend class ScalarEvolution; @@ -393,10 +402,6 @@ namespace llvm { class SCEVUnknown final : public SCEV, private CallbackVH { friend class ScalarEvolution; - // Implement CallbackVH. - void deleted() override; - void allUsesReplacedWith(Value *New) override; - /// The parent ScalarEvolution value. This is used to update the /// parent's maps when the value associated with a SCEVUnknown is /// deleted or RAUW'd. @@ -410,6 +415,10 @@ namespace llvm { ScalarEvolution *se, SCEVUnknown *next) : SCEV(ID, scUnknown), CallbackVH(V), SE(se), Next(next) {} + // Implement CallbackVH. + void deleted() override; + void allUsesReplacedWith(Value *New) override; + public: Value *getValue() const { return getValPtr(); } @@ -490,6 +499,7 @@ namespace llvm { if (Visited.insert(S).second && Visitor.follow(S)) Worklist.push_back(S); } + public: SCEVTraversal(SV& V): Visitor(V) {} @@ -682,7 +692,7 @@ namespace llvm { } }; - typedef DenseMap<const Value*, Value*> ValueToValueMap; + using ValueToValueMap = DenseMap<const Value *, Value *>; /// The SCEVParameterRewriter takes a scalar evolution expression and updates /// the SCEVUnknown components following the Map (Value -> Value). @@ -714,26 +724,26 @@ namespace llvm { bool InterpretConsts; }; - typedef DenseMap<const Loop*, const SCEV*> LoopToScevMapT; + using LoopToScevMapT = DenseMap<const Loop *, const SCEV *>; /// The SCEVLoopAddRecRewriter takes a scalar evolution expression and applies /// the Map (Loop -> SCEV) to all AddRecExprs. class SCEVLoopAddRecRewriter : public SCEVRewriteVisitor<SCEVLoopAddRecRewriter> { public: + SCEVLoopAddRecRewriter(ScalarEvolution &SE, LoopToScevMapT &M) + : SCEVRewriteVisitor(SE), Map(M) {} + static const SCEV *rewrite(const SCEV *Scev, LoopToScevMapT &Map, ScalarEvolution &SE) { SCEVLoopAddRecRewriter Rewriter(SE, Map); return Rewriter.visit(Scev); } - SCEVLoopAddRecRewriter(ScalarEvolution &SE, LoopToScevMapT &M) - : SCEVRewriteVisitor(SE), Map(M) {} - const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { SmallVector<const SCEV *, 2> Operands; - for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + for (const SCEV *Op : Expr->operands()) + Operands.push_back(visit(Op)); const Loop *L = Expr->getLoop(); const SCEV *Res = SE.getAddRecExpr(Operands, L, Expr->getNoWrapFlags()); @@ -748,6 +758,7 @@ namespace llvm { private: LoopToScevMapT ⤅ }; -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H diff --git a/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h b/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h index a7b57310d2d0..508968e16e5d 100644 --- a/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h +++ b/contrib/llvm/include/llvm/Analysis/ScopedNoAliasAA.h @@ -6,22 +6,27 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This is the interface for a metadata-based scoped no-alias analysis. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_SCOPEDNOALIASAA_H #define LLVM_ANALYSIS_SCOPEDNOALIASAA_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/Module.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include <memory> namespace llvm { +class Function; +class MDNode; +class MemoryLocation; + /// A simple AA result which uses scoped-noalias metadata to answer queries. class ScopedNoAliasAAResult : public AAResultBase<ScopedNoAliasAAResult> { friend AAResultBase<ScopedNoAliasAAResult>; @@ -46,10 +51,11 @@ private: /// Analysis pass providing a never-invalidated alias analysis result. class ScopedNoAliasAA : public AnalysisInfoMixin<ScopedNoAliasAA> { friend AnalysisInfoMixin<ScopedNoAliasAA>; + static AnalysisKey Key; public: - typedef ScopedNoAliasAAResult Result; + using Result = ScopedNoAliasAAResult; ScopedNoAliasAAResult run(Function &F, FunctionAnalysisManager &AM); }; @@ -77,6 +83,7 @@ public: // scoped noalias analysis. // ImmutablePass *createScopedNoAliasAAWrapperPass(); -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_SCOPEDNOALIASAA_H diff --git a/contrib/llvm/include/llvm/Analysis/SparsePropagation.h b/contrib/llvm/include/llvm/Analysis/SparsePropagation.h index d1a54171d8bd..1b8df03b3a1b 100644 --- a/contrib/llvm/include/llvm/Analysis/SparsePropagation.h +++ b/contrib/llvm/include/llvm/Analysis/SparsePropagation.h @@ -15,37 +15,35 @@ #ifndef LLVM_ANALYSIS_SPARSEPROPAGATION_H #define LLVM_ANALYSIS_SPARSEPROPAGATION_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Support/Debug.h" #include <set> -#include <vector> + +#define DEBUG_TYPE "sparseprop" namespace llvm { -class Value; -class Constant; -class Argument; -class Instruction; -class PHINode; -class TerminatorInst; -class BasicBlock; -class Function; -class SparseSolver; -class raw_ostream; -template <typename T> class SmallVectorImpl; +/// A template for translating between LLVM Values and LatticeKeys. Clients must +/// provide a specialization of LatticeKeyInfo for their LatticeKey type. +template <class LatticeKey> struct LatticeKeyInfo { + // static inline Value *getValueFromLatticeKey(LatticeKey Key); + // static inline LatticeKey getLatticeKeyFromValue(Value *V); +}; -/// AbstractLatticeFunction - This class is implemented by the dataflow instance -/// to specify what the lattice values are and how they handle merges etc. -/// This gives the client the power to compute lattice values from instructions, -/// constants, etc. The requirement is that lattice values must all fit into -/// a void*. If a void* is not sufficient, the implementation should use this -/// pointer to be a pointer into a uniquing set or something. -/// -class AbstractLatticeFunction { -public: - typedef void *LatticeVal; +template <class LatticeKey, class LatticeVal, + class KeyInfo = LatticeKeyInfo<LatticeKey>> +class SparseSolver; +/// AbstractLatticeFunction - This class is implemented by the dataflow instance +/// to specify what the lattice values are and how they handle merges etc. This +/// gives the client the power to compute lattice values from instructions, +/// constants, etc. The current requirement is that lattice values must be +/// copyable. At the moment, nothing tries to avoid copying. Additionally, +/// lattice keys must be able to be used as keys of a mapping data structure. +/// Internally, the generic solver currently uses a DenseMap to map lattice keys +/// to lattice values. If the lattice key is a non-standard type, a +/// specialization of DenseMapInfo must be provided. +template <class LatticeKey, class LatticeVal> class AbstractLatticeFunction { private: LatticeVal UndefVal, OverdefinedVal, UntrackedVal; @@ -56,40 +54,28 @@ public: OverdefinedVal = overdefinedVal; UntrackedVal = untrackedVal; } - virtual ~AbstractLatticeFunction(); + + virtual ~AbstractLatticeFunction() = default; LatticeVal getUndefVal() const { return UndefVal; } LatticeVal getOverdefinedVal() const { return OverdefinedVal; } LatticeVal getUntrackedVal() const { return UntrackedVal; } - /// IsUntrackedValue - If the specified Value is something that is obviously - /// uninteresting to the analysis (and would always return UntrackedVal), - /// this function can return true to avoid pointless work. - virtual bool IsUntrackedValue(Value *V) { return false; } + /// IsUntrackedValue - If the specified LatticeKey is obviously uninteresting + /// to the analysis (i.e., it would always return UntrackedVal), this + /// function can return true to avoid pointless work. + virtual bool IsUntrackedValue(LatticeKey Key) { return false; } - /// ComputeConstant - Given a constant value, compute and return a lattice - /// value corresponding to the specified constant. - virtual LatticeVal ComputeConstant(Constant *C) { - return getOverdefinedVal(); // always safe + /// ComputeLatticeVal - Compute and return a LatticeVal corresponding to the + /// given LatticeKey. + virtual LatticeVal ComputeLatticeVal(LatticeKey Key) { + return getOverdefinedVal(); } /// IsSpecialCasedPHI - Given a PHI node, determine whether this PHI node is /// one that the we want to handle through ComputeInstructionState. virtual bool IsSpecialCasedPHI(PHINode *PN) { return false; } - /// GetConstant - If the specified lattice value is representable as an LLVM - /// constant value, return it. Otherwise return null. The returned value - /// must be in the same LLVM type as Val. - virtual Constant *GetConstant(LatticeVal LV, Value *Val, SparseSolver &SS) { - return nullptr; - } - - /// ComputeArgument - Given a formal argument value, compute and return a - /// lattice value corresponding to the specified argument. - virtual LatticeVal ComputeArgument(Argument *I) { - return getOverdefinedVal(); // always safe - } - /// MergeValues - Compute and return the merge of the two specified lattice /// values. Merging should only move one direction down the lattice to /// guarantee convergence (toward overdefined). @@ -97,67 +83,80 @@ public: return getOverdefinedVal(); // always safe, never useful. } - /// ComputeInstructionState - Given an instruction and a vector of its operand - /// values, compute the result value of the instruction. - virtual LatticeVal ComputeInstructionState(Instruction &I, SparseSolver &SS) { - return getOverdefinedVal(); // always safe, never useful. + /// ComputeInstructionState - Compute the LatticeKeys that change as a result + /// of executing instruction \p I. Their associated LatticeVals are store in + /// \p ChangedValues. + virtual void + ComputeInstructionState(Instruction &I, + DenseMap<LatticeKey, LatticeVal> &ChangedValues, + SparseSolver<LatticeKey, LatticeVal> &SS) = 0; + + /// PrintLatticeVal - Render the given LatticeVal to the specified stream. + virtual void PrintLatticeVal(LatticeVal LV, raw_ostream &OS); + + /// PrintLatticeKey - Render the given LatticeKey to the specified stream. + virtual void PrintLatticeKey(LatticeKey Key, raw_ostream &OS); + + /// GetValueFromLatticeVal - If the given LatticeVal is representable as an + /// LLVM value, return it; otherwise, return nullptr. If a type is given, the + /// returned value must have the same type. This function is used by the + /// generic solver in attempting to resolve branch and switch conditions. + virtual Value *GetValueFromLatticeVal(LatticeVal LV, Type *Ty = nullptr) { + return nullptr; } - - /// PrintValue - Render the specified lattice value to the specified stream. - virtual void PrintValue(LatticeVal V, raw_ostream &OS); }; /// SparseSolver - This class is a general purpose solver for Sparse Conditional /// Propagation with a programmable lattice function. -/// +template <class LatticeKey, class LatticeVal, class KeyInfo> class SparseSolver { - typedef AbstractLatticeFunction::LatticeVal LatticeVal; - /// LatticeFunc - This is the object that knows the lattice and how to do + /// LatticeFunc - This is the object that knows the lattice and how to /// compute transfer functions. - AbstractLatticeFunction *LatticeFunc; + AbstractLatticeFunction<LatticeKey, LatticeVal> *LatticeFunc; + + /// ValueState - Holds the LatticeVals associated with LatticeKeys. + DenseMap<LatticeKey, LatticeVal> ValueState; - DenseMap<Value *, LatticeVal> ValueState; // The state each value is in. - SmallPtrSet<BasicBlock *, 16> BBExecutable; // The bbs that are executable. + /// BBExecutable - Holds the basic blocks that are executable. + SmallPtrSet<BasicBlock *, 16> BBExecutable; - std::vector<Instruction *> InstWorkList; // Worklist of insts to process. + /// ValueWorkList - Holds values that should be processed. + SmallVector<Value *, 64> ValueWorkList; - std::vector<BasicBlock *> BBWorkList; // The BasicBlock work list + /// BBWorkList - Holds basic blocks that should be processed. + SmallVector<BasicBlock *, 64> BBWorkList; + + using Edge = std::pair<BasicBlock *, BasicBlock *>; /// KnownFeasibleEdges - Entries in this set are edges which have already had /// PHI nodes retriggered. - typedef std::pair<BasicBlock*,BasicBlock*> Edge; std::set<Edge> KnownFeasibleEdges; - SparseSolver(const SparseSolver&) = delete; - void operator=(const SparseSolver&) = delete; - public: - explicit SparseSolver(AbstractLatticeFunction *Lattice) + explicit SparseSolver( + AbstractLatticeFunction<LatticeKey, LatticeVal> *Lattice) : LatticeFunc(Lattice) {} - ~SparseSolver() { delete LatticeFunc; } + SparseSolver(const SparseSolver &) = delete; + SparseSolver &operator=(const SparseSolver &) = delete; /// Solve - Solve for constants and executable blocks. - /// - void Solve(Function &F); + void Solve(); - void Print(Function &F, raw_ostream &OS) const; + void Print(raw_ostream &OS) const; - /// getLatticeState - Return the LatticeVal object that corresponds to the - /// value. If an value is not in the map, it is returned as untracked, - /// unlike the getOrInitValueState method. - LatticeVal getLatticeState(Value *V) const { - DenseMap<Value*, LatticeVal>::const_iterator I = ValueState.find(V); + /// getExistingValueState - Return the LatticeVal object corresponding to the + /// given value from the ValueState map. If the value is not in the map, + /// UntrackedVal is returned, unlike the getValueState method. + LatticeVal getExistingValueState(LatticeKey Key) const { + auto I = ValueState.find(Key); return I != ValueState.end() ? I->second : LatticeFunc->getUntrackedVal(); } - /// getOrInitValueState - Return the LatticeVal object that corresponds to the - /// value, initializing the value's state if it hasn't been entered into the - /// map yet. This function is necessary because not all values should start - /// out in the underdefined state... Arguments should be overdefined, and - /// constants should be marked as constants. - /// - LatticeVal getOrInitValueState(Value *V); + /// getValueState - Return the LatticeVal object corresponding to the given + /// value from the ValueState map. If the value is not in the map, its state + /// is initialized. + LatticeVal getValueState(LatticeKey Key); /// isEdgeFeasible - Return true if the control flow edge from the 'From' /// basic block to the 'To' basic block is currently feasible. If @@ -174,15 +173,16 @@ public: return BBExecutable.count(BB); } -private: - /// UpdateState - When the state for some instruction is potentially updated, - /// this function notices and adds I to the worklist if needed. - void UpdateState(Instruction &Inst, LatticeVal V); - /// MarkBlockExecutable - This method can be used by clients to mark all of /// the blocks that are known to be intrinsically live in the processed unit. void MarkBlockExecutable(BasicBlock *BB); +private: + /// UpdateState - When the state of some LatticeKey is potentially updated to + /// the given LatticeVal, this function notices and adds the LLVM value + /// corresponding the key to the work list, if needed. + void UpdateState(LatticeKey Key, LatticeVal LV); + /// markEdgeExecutable - Mark a basic block as executable, adding it to the BB /// work list if it is not already executable. void markEdgeExecutable(BasicBlock *Source, BasicBlock *Dest); @@ -197,6 +197,334 @@ private: void visitTerminatorInst(TerminatorInst &TI); }; +//===----------------------------------------------------------------------===// +// AbstractLatticeFunction Implementation +//===----------------------------------------------------------------------===// + +template <class LatticeKey, class LatticeVal> +void AbstractLatticeFunction<LatticeKey, LatticeVal>::PrintLatticeVal( + LatticeVal V, raw_ostream &OS) { + if (V == UndefVal) + OS << "undefined"; + else if (V == OverdefinedVal) + OS << "overdefined"; + else if (V == UntrackedVal) + OS << "untracked"; + else + OS << "unknown lattice value"; +} + +template <class LatticeKey, class LatticeVal> +void AbstractLatticeFunction<LatticeKey, LatticeVal>::PrintLatticeKey( + LatticeKey Key, raw_ostream &OS) { + OS << "unknown lattice key"; +} + +//===----------------------------------------------------------------------===// +// SparseSolver Implementation +//===----------------------------------------------------------------------===// + +template <class LatticeKey, class LatticeVal, class KeyInfo> +LatticeVal +SparseSolver<LatticeKey, LatticeVal, KeyInfo>::getValueState(LatticeKey Key) { + auto I = ValueState.find(Key); + if (I != ValueState.end()) + return I->second; // Common case, in the map + + if (LatticeFunc->IsUntrackedValue(Key)) + return LatticeFunc->getUntrackedVal(); + LatticeVal LV = LatticeFunc->ComputeLatticeVal(Key); + + // If this value is untracked, don't add it to the map. + if (LV == LatticeFunc->getUntrackedVal()) + return LV; + return ValueState[Key] = LV; +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::UpdateState(LatticeKey Key, + LatticeVal LV) { + auto I = ValueState.find(Key); + if (I != ValueState.end() && I->second == LV) + return; // No change. + + // Update the state of the given LatticeKey and add its corresponding LLVM + // value to the work list. + ValueState[Key] = LV; + if (Value *V = KeyInfo::getValueFromLatticeKey(Key)) + ValueWorkList.push_back(V); +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::MarkBlockExecutable( + BasicBlock *BB) { + if (!BBExecutable.insert(BB).second) + return; + DEBUG(dbgs() << "Marking Block Executable: " << BB->getName() << "\n"); + BBWorkList.push_back(BB); // Add the block to the work list! +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::markEdgeExecutable( + BasicBlock *Source, BasicBlock *Dest) { + if (!KnownFeasibleEdges.insert(Edge(Source, Dest)).second) + return; // This edge is already known to be executable! + + DEBUG(dbgs() << "Marking Edge Executable: " << Source->getName() << " -> " + << Dest->getName() << "\n"); + + if (BBExecutable.count(Dest)) { + // The destination is already executable, but we just made an edge + // feasible that wasn't before. Revisit the PHI nodes in the block + // because they have potentially new operands. + for (BasicBlock::iterator I = Dest->begin(); isa<PHINode>(I); ++I) + visitPHINode(*cast<PHINode>(I)); + } else { + MarkBlockExecutable(Dest); + } +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::getFeasibleSuccessors( + TerminatorInst &TI, SmallVectorImpl<bool> &Succs, bool AggressiveUndef) { + Succs.resize(TI.getNumSuccessors()); + if (TI.getNumSuccessors() == 0) + return; + + if (BranchInst *BI = dyn_cast<BranchInst>(&TI)) { + if (BI->isUnconditional()) { + Succs[0] = true; + return; + } + + LatticeVal BCValue; + if (AggressiveUndef) + BCValue = + getValueState(KeyInfo::getLatticeKeyFromValue(BI->getCondition())); + else + BCValue = getExistingValueState( + KeyInfo::getLatticeKeyFromValue(BI->getCondition())); + + if (BCValue == LatticeFunc->getOverdefinedVal() || + BCValue == LatticeFunc->getUntrackedVal()) { + // Overdefined condition variables can branch either way. + Succs[0] = Succs[1] = true; + return; + } + + // If undefined, neither is feasible yet. + if (BCValue == LatticeFunc->getUndefVal()) + return; + + Constant *C = + dyn_cast_or_null<Constant>(LatticeFunc->GetValueFromLatticeVal( + BCValue, BI->getCondition()->getType())); + if (!C || !isa<ConstantInt>(C)) { + // Non-constant values can go either way. + Succs[0] = Succs[1] = true; + return; + } + + // Constant condition variables mean the branch can only go a single way + Succs[C->isNullValue()] = true; + return; + } + + if (TI.isExceptional()) { + Succs.assign(Succs.size(), true); + return; + } + + if (isa<IndirectBrInst>(TI)) { + Succs.assign(Succs.size(), true); + return; + } + + SwitchInst &SI = cast<SwitchInst>(TI); + LatticeVal SCValue; + if (AggressiveUndef) + SCValue = getValueState(KeyInfo::getLatticeKeyFromValue(SI.getCondition())); + else + SCValue = getExistingValueState( + KeyInfo::getLatticeKeyFromValue(SI.getCondition())); + + if (SCValue == LatticeFunc->getOverdefinedVal() || + SCValue == LatticeFunc->getUntrackedVal()) { + // All destinations are executable! + Succs.assign(TI.getNumSuccessors(), true); + return; + } + + // If undefined, neither is feasible yet. + if (SCValue == LatticeFunc->getUndefVal()) + return; + + Constant *C = dyn_cast_or_null<Constant>(LatticeFunc->GetValueFromLatticeVal( + SCValue, SI.getCondition()->getType())); + if (!C || !isa<ConstantInt>(C)) { + // All destinations are executable! + Succs.assign(TI.getNumSuccessors(), true); + return; + } + SwitchInst::CaseHandle Case = *SI.findCaseValue(cast<ConstantInt>(C)); + Succs[Case.getSuccessorIndex()] = true; +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +bool SparseSolver<LatticeKey, LatticeVal, KeyInfo>::isEdgeFeasible( + BasicBlock *From, BasicBlock *To, bool AggressiveUndef) { + SmallVector<bool, 16> SuccFeasible; + TerminatorInst *TI = From->getTerminator(); + getFeasibleSuccessors(*TI, SuccFeasible, AggressiveUndef); + + for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) + if (TI->getSuccessor(i) == To && SuccFeasible[i]) + return true; + + return false; +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::visitTerminatorInst( + TerminatorInst &TI) { + SmallVector<bool, 16> SuccFeasible; + getFeasibleSuccessors(TI, SuccFeasible, true); + + BasicBlock *BB = TI.getParent(); + + // Mark all feasible successors executable... + for (unsigned i = 0, e = SuccFeasible.size(); i != e; ++i) + if (SuccFeasible[i]) + markEdgeExecutable(BB, TI.getSuccessor(i)); +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::visitPHINode(PHINode &PN) { + // The lattice function may store more information on a PHINode than could be + // computed from its incoming values. For example, SSI form stores its sigma + // functions as PHINodes with a single incoming value. + if (LatticeFunc->IsSpecialCasedPHI(&PN)) { + DenseMap<LatticeKey, LatticeVal> ChangedValues; + LatticeFunc->ComputeInstructionState(PN, ChangedValues, *this); + for (auto &ChangedValue : ChangedValues) + if (ChangedValue.second != LatticeFunc->getUntrackedVal()) + UpdateState(ChangedValue.first, ChangedValue.second); + return; + } + + LatticeKey Key = KeyInfo::getLatticeKeyFromValue(&PN); + LatticeVal PNIV = getValueState(Key); + LatticeVal Overdefined = LatticeFunc->getOverdefinedVal(); + + // If this value is already overdefined (common) just return. + if (PNIV == Overdefined || PNIV == LatticeFunc->getUntrackedVal()) + return; // Quick exit + + // Super-extra-high-degree PHI nodes are unlikely to ever be interesting, + // and slow us down a lot. Just mark them overdefined. + if (PN.getNumIncomingValues() > 64) { + UpdateState(Key, Overdefined); + return; + } + + // Look at all of the executable operands of the PHI node. If any of them + // are overdefined, the PHI becomes overdefined as well. Otherwise, ask the + // transfer function to give us the merge of the incoming values. + for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) { + // If the edge is not yet known to be feasible, it doesn't impact the PHI. + if (!isEdgeFeasible(PN.getIncomingBlock(i), PN.getParent(), true)) + continue; + + // Merge in this value. + LatticeVal OpVal = + getValueState(KeyInfo::getLatticeKeyFromValue(PN.getIncomingValue(i))); + if (OpVal != PNIV) + PNIV = LatticeFunc->MergeValues(PNIV, OpVal); + + if (PNIV == Overdefined) + break; // Rest of input values don't matter. + } + + // Update the PHI with the compute value, which is the merge of the inputs. + UpdateState(Key, PNIV); +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::visitInst(Instruction &I) { + // PHIs are handled by the propagation logic, they are never passed into the + // transfer functions. + if (PHINode *PN = dyn_cast<PHINode>(&I)) + return visitPHINode(*PN); + + // Otherwise, ask the transfer function what the result is. If this is + // something that we care about, remember it. + DenseMap<LatticeKey, LatticeVal> ChangedValues; + LatticeFunc->ComputeInstructionState(I, ChangedValues, *this); + for (auto &ChangedValue : ChangedValues) + if (ChangedValue.second != LatticeFunc->getUntrackedVal()) + UpdateState(ChangedValue.first, ChangedValue.second); + + if (TerminatorInst *TI = dyn_cast<TerminatorInst>(&I)) + visitTerminatorInst(*TI); +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::Solve() { + // Process the work lists until they are empty! + while (!BBWorkList.empty() || !ValueWorkList.empty()) { + // Process the value work list. + while (!ValueWorkList.empty()) { + Value *V = ValueWorkList.back(); + ValueWorkList.pop_back(); + + DEBUG(dbgs() << "\nPopped off V-WL: " << *V << "\n"); + + // "V" got into the work list because it made a transition. See if any + // users are both live and in need of updating. + for (User *U : V->users()) + if (Instruction *Inst = dyn_cast<Instruction>(U)) + if (BBExecutable.count(Inst->getParent())) // Inst is executable? + visitInst(*Inst); + } + + // Process the basic block work list. + while (!BBWorkList.empty()) { + BasicBlock *BB = BBWorkList.back(); + BBWorkList.pop_back(); + + DEBUG(dbgs() << "\nPopped off BBWL: " << *BB); + + // Notify all instructions in this basic block that they are newly + // executable. + for (Instruction &I : *BB) + visitInst(I); + } + } +} + +template <class LatticeKey, class LatticeVal, class KeyInfo> +void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::Print( + raw_ostream &OS) const { + if (ValueState.empty()) + return; + + LatticeKey Key; + LatticeVal LV; + + OS << "ValueState:\n"; + for (auto &Entry : ValueState) { + std::tie(Key, LV) = Entry; + if (LV == LatticeFunc->getUntrackedVal()) + continue; + OS << "\t"; + LatticeFunc->PrintLatticeVal(LV, OS); + OS << ": "; + LatticeFunc->PrintLatticeKey(Key, OS); + OS << "\n"; + } +} } // end namespace llvm +#undef DEBUG_TYPE + #endif // LLVM_ANALYSIS_SPARSEPROPAGATION_H diff --git a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def index 9cbe917c146d..a461ed813b9b 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def +++ b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.def @@ -457,6 +457,15 @@ TLI_DEFINE_STRING_INTERNAL("bcopy") /// void bzero(void *s, size_t n); TLI_DEFINE_ENUM_INTERNAL(bzero) TLI_DEFINE_STRING_INTERNAL("bzero") +/// double cabs(double complex z) +TLI_DEFINE_ENUM_INTERNAL(cabs) +TLI_DEFINE_STRING_INTERNAL("cabs") +/// float cabs(float complex z) +TLI_DEFINE_ENUM_INTERNAL(cabsf) +TLI_DEFINE_STRING_INTERNAL("cabsf") +/// long double cabs(long double complex z) +TLI_DEFINE_ENUM_INTERNAL(cabsl) +TLI_DEFINE_STRING_INTERNAL("cabsl") /// void *calloc(size_t count, size_t size); TLI_DEFINE_ENUM_INTERNAL(calloc) TLI_DEFINE_STRING_INTERNAL("calloc") diff --git a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.h index d75e7833279b..a3fe834022f7 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/contrib/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -193,13 +193,9 @@ public: ShouldSignExtI32Param = Val; } - /// Returns the size of the wchar_t type in bytes. + /// Returns the size of the wchar_t type in bytes or 0 if the size is unknown. + /// This queries the 'wchar_size' metadata. unsigned getWCharSize(const Module &M) const; - - /// Returns size of the default wchar_t type on target \p T. This is mostly - /// intended to verify that the size in the frontend matches LLVM. All other - /// queries should use getWCharSize() instead. - static unsigned getTargetWCharSize(const Triple &T); }; /// Provides information about what library functions are available for diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h index 24edd3826a2e..c20f20cfbe4d 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -23,21 +23,28 @@ #define LLVM_ANALYSIS_TARGETTRANSFORMINFO_H #include "llvm/ADT/Optional.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" #include <functional> namespace llvm { +namespace Intrinsic { +enum ID : unsigned; +} + class Function; class GlobalValue; +class IntrinsicInst; +class LoadInst; class Loop; -class ScalarEvolution; class SCEV; +class ScalarEvolution; +class StoreInst; +class SwitchInst; class Type; class User; class Value; @@ -107,6 +114,37 @@ public: /// \name Generic Target Information /// @{ + /// \brief The kind of cost model. + /// + /// There are several different cost models that can be customized by the + /// target. The normalization of each cost model may be target specific. + enum TargetCostKind { + TCK_RecipThroughput, ///< Reciprocal throughput. + TCK_Latency, ///< The latency of instruction. + TCK_CodeSize ///< Instruction code size. + }; + + /// \brief Query the cost of a specified instruction. + /// + /// Clients should use this interface to query the cost of an existing + /// instruction. The instruction must have a valid parent (basic block). + /// + /// Note, this method does not cache the cost calculation and it + /// can be expensive in some cases. + int getInstructionCost(const Instruction *I, enum TargetCostKind kind) const { + switch (kind){ + case TCK_RecipThroughput: + return getInstructionThroughput(I); + + case TCK_Latency: + return getInstructionLatency(I); + + case TCK_CodeSize: + return getUserCost(I); + } + llvm_unreachable("Unknown instruction cost kind"); + } + /// \brief Underlying constants for 'cost' values in this interface. /// /// Many APIs in this interface return a cost. This enum defines the @@ -382,6 +420,8 @@ public: bool UpperBound; /// Allow peeling off loop iterations for loops with low dynamic tripcount. bool AllowPeeling; + /// Allow unrolling of all the iterations of the runtime loop remainder. + bool UnrollRemainder; }; /// \brief Get target-customized preferences for the generic loop unrolling @@ -420,10 +460,12 @@ public: /// this target, for a load/store of the specified type. /// The type may be VoidTy, in which case only return true if the addressing /// mode is legal for a load/store of any legal type. + /// If target returns true in LSRWithInstrQueries(), I may be valid. /// TODO: Handle pre/postinc as well. bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, - unsigned AddrSpace = 0) const; + unsigned AddrSpace = 0, + Instruction *I = nullptr) const; /// \brief Return true if LSR cost of C1 is lower than C1. bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, @@ -440,6 +482,20 @@ public: bool isLegalMaskedScatter(Type *DataType) const; bool isLegalMaskedGather(Type *DataType) const; + /// Return true if the target has a unified operation to calculate division + /// and remainder. If so, the additional implicit multiplication and + /// subtraction required to calculate a remainder from division are free. This + /// can enable more aggressive transformations for division and remainder than + /// would typically be allowed using throughput or size cost models. + bool hasDivRemOp(Type *DataType, bool IsSigned) const; + + /// Return true if the given instruction (assumed to be a memory access + /// instruction) has a volatile variant. If that's the case then we can avoid + /// addrspacecast to generic AS for volatile loads/stores. Default + /// implementation returns false, which prevents address space inference for + /// volatile loads/stores. + bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const; + /// Return true if target doesn't mind addresses in vectors. bool prefersVectorizedAddressing() const; @@ -453,12 +509,12 @@ public: bool HasBaseReg, int64_t Scale, unsigned AddrSpace = 0) const; - /// \brief Return true if target supports the load / store - /// instruction with the given Offset on the form reg + Offset. It - /// may be that Offset is too big for a certain type (register - /// class). - bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) const; - + /// \brief Return true if the loop strength reduce pass should make + /// Instruction* based TTI queries to isLegalAddressingMode(). This is + /// needed on SystemZ, where e.g. a memcpy can only have a 12 bit unsigned + /// immediate offset and no index register. + bool LSRWithInstrQueries() const; + /// \brief Return true if it's free to truncate a value of type Ty1 to type /// Ty2. e.g. On x86 it's free to truncate a i32 value in register EAX to i16 /// by referencing its sub-register AX. @@ -498,8 +554,13 @@ public: /// \brief Don't restrict interleaved unrolling to small loops. bool enableAggressiveInterleaving(bool LoopHasReductions) const; - /// \brief Enable inline expansion of memcmp - bool expandMemCmp(Instruction *I, unsigned &MaxLoadSize) const; + /// \brief If not nullptr, enable inline expansion of memcmp. IsZeroCmp is + /// true if this is the expansion of memcmp(p1, p2, s) == 0. + struct MemCmpExpansionOptions { + // The list of available load sizes (in bytes), sorted in decreasing order. + SmallVector<unsigned, 8> LoadSizes; + }; + const MemCmpExpansionOptions *enableMemCmpExpansion(bool IsZeroCmp) const; /// \brief Enable matching of interleaved access groups. bool enableInterleavedAccessVectorization() const; @@ -525,6 +586,12 @@ public: /// \brief Return true if the hardware has a fast square-root instruction. bool haveFastSqrt(Type *Ty) const; + /// Return true if it is faster to check if a floating-point value is NaN + /// (or not-NaN) versus a comparison against a constant FP zero value. + /// Targets should override this if materializing a 0.0 for comparison is + /// generally as cheap as checking for ordered/unordered. + bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const; + /// \brief Return the expected cost of supporting the floating point operation /// of the specified type. int getFPOpCost(Type *Ty) const; @@ -599,6 +666,22 @@ public: /// \return The size of a cache line in bytes. unsigned getCacheLineSize() const; + /// The possible cache levels + enum class CacheLevel { + L1D, // The L1 data cache + L2D, // The L2 data cache + + // We currently do not model L3 caches, as their sizes differ widely between + // microarchitectures. Also, we currently do not have a use for L3 cache + // size modeling yet. + }; + + /// \return The size of the cache level in bytes, if available. + llvm::Optional<unsigned> getCacheSize(CacheLevel Level) const; + + /// \return The associativity of the cache level, if available. + llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) const; + /// \return How much before a load we should place the prefetch instruction. /// This is currently measured in number of instructions. unsigned getPrefetchDistance() const; @@ -699,11 +782,14 @@ public: /// /// Pairwise: /// (v0, v1, v2, v3) - /// ((v0+v1), (v2, v3), undef, undef) + /// ((v0+v1), (v2+v3), undef, undef) /// Split: /// (v0, v1, v2, v3) /// ((v0+v2), (v1+v3), undef, undef) - int getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwiseForm) const; + int getArithmeticReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) const; + int getMinMaxReductionCost(Type *Ty, Type *CondTy, bool IsPairwiseForm, + bool IsUnsigned) const; /// \returns The cost of Intrinsic instructions. Analyses the real arguments. /// Three cases are handled: 1. scalar instruction 2. vector instruction @@ -776,12 +862,6 @@ public: unsigned SrcAlign, unsigned DestAlign) const; - /// \returns True if we want to test the new memcpy lowering functionality in - /// Transform/Utils. - /// Temporary. Will be removed once we move to the new functionality and - /// remove the old. - bool useWideIRMemcpyLoopLowering() const; - /// \returns True if the two functions have compatible attributes for inlining /// purposes. bool areInlineCompatible(const Function *Caller, @@ -838,6 +918,14 @@ public: /// @} private: + /// \brief Estimate the latency of specified instruction. + /// Returns 1 as the default value. + int getInstructionLatency(const Instruction *I) const; + + /// \brief Returns the expected throughput cost of the instruction. + /// Returns -1 if the cost is unknown. + int getInstructionThroughput(const Instruction *I) const; + /// \brief The abstract base class used to type erase specific TTI /// implementations. class Concept; @@ -882,18 +970,21 @@ public: virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, - unsigned AddrSpace) = 0; + unsigned AddrSpace, + Instruction *I) = 0; virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) = 0; virtual bool isLegalMaskedStore(Type *DataType) = 0; virtual bool isLegalMaskedLoad(Type *DataType) = 0; virtual bool isLegalMaskedScatter(Type *DataType) = 0; virtual bool isLegalMaskedGather(Type *DataType) = 0; + virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; + virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; virtual bool prefersVectorizedAddressing() = 0; virtual int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) = 0; - virtual bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) = 0; + virtual bool LSRWithInstrQueries() = 0; virtual bool isTruncateFree(Type *Ty1, Type *Ty2) = 0; virtual bool isProfitableToHoist(Instruction *I) = 0; virtual bool isTypeLegal(Type *Ty) = 0; @@ -907,7 +998,8 @@ public: unsigned VF) = 0; virtual bool supportsEfficientVectorElementLoadStore() = 0; virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0; - virtual bool expandMemCmp(Instruction *I, unsigned &MaxLoadSize) = 0; + virtual const MemCmpExpansionOptions *enableMemCmpExpansion( + bool IsZeroCmp) const = 0; virtual bool enableInterleavedAccessVectorization() = 0; virtual bool isFPVectorizationPotentiallyUnsafe() = 0; virtual bool allowsMisalignedMemoryAccesses(LLVMContext &Context, @@ -917,6 +1009,7 @@ public: bool *Fast) = 0; virtual PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) = 0; virtual bool haveFastSqrt(Type *Ty) = 0; + virtual bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) = 0; virtual int getFPOpCost(Type *Ty) = 0; virtual int getIntImmCodeSizeCost(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty) = 0; @@ -931,6 +1024,8 @@ public: virtual bool shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) = 0; virtual unsigned getCacheLineSize() = 0; + virtual llvm::Optional<unsigned> getCacheSize(CacheLevel Level) = 0; + virtual llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) = 0; virtual unsigned getPrefetchDistance() = 0; virtual unsigned getMinPrefetchStride() = 0; virtual unsigned getMaxPrefetchIterationsAhead() = 0; @@ -965,8 +1060,10 @@ public: ArrayRef<unsigned> Indices, unsigned Alignment, unsigned AddressSpace) = 0; - virtual int getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) = 0; + virtual int getArithmeticReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) = 0; + virtual int getMinMaxReductionCost(Type *Ty, Type *CondTy, + bool IsPairwiseForm, bool IsUnsigned) = 0; virtual int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, ArrayRef<Type *> Tys, FastMathFlags FMF, unsigned ScalarizationCostPassed) = 0; @@ -1009,6 +1106,7 @@ public: virtual bool useReductionIntrinsic(unsigned Opcode, Type *Ty, ReductionFlags) const = 0; virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0; + virtual int getInstructionLatency(const Instruction *I) = 0; }; template <typename T> @@ -1085,9 +1183,10 @@ public: } bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, - unsigned AddrSpace) override { + unsigned AddrSpace, + Instruction *I) override { return Impl.isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, - Scale, AddrSpace); + Scale, AddrSpace, I); } bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) override { @@ -1105,6 +1204,12 @@ public: bool isLegalMaskedGather(Type *DataType) override { return Impl.isLegalMaskedGather(DataType); } + bool hasDivRemOp(Type *DataType, bool IsSigned) override { + return Impl.hasDivRemOp(DataType, IsSigned); + } + bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) override { + return Impl.hasVolatileVariant(I, AddrSpace); + } bool prefersVectorizedAddressing() override { return Impl.prefersVectorizedAddressing(); } @@ -1114,8 +1219,8 @@ public: return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); } - bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) override { - return Impl.isFoldableMemAccessOffset(I, Offset); + bool LSRWithInstrQueries() override { + return Impl.LSRWithInstrQueries(); } bool isTruncateFree(Type *Ty1, Type *Ty2) override { return Impl.isTruncateFree(Ty1, Ty2); @@ -1148,8 +1253,9 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) override { return Impl.enableAggressiveInterleaving(LoopHasReductions); } - bool expandMemCmp(Instruction *I, unsigned &MaxLoadSize) override { - return Impl.expandMemCmp(I, MaxLoadSize); + const MemCmpExpansionOptions *enableMemCmpExpansion( + bool IsZeroCmp) const override { + return Impl.enableMemCmpExpansion(IsZeroCmp); } bool enableInterleavedAccessVectorization() override { return Impl.enableInterleavedAccessVectorization(); @@ -1168,6 +1274,10 @@ public: } bool haveFastSqrt(Type *Ty) override { return Impl.haveFastSqrt(Ty); } + bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) override { + return Impl.isFCmpOrdCheaperThanFCmpZero(Ty); + } + int getFPOpCost(Type *Ty) override { return Impl.getFPOpCost(Ty); } int getIntImmCodeSizeCost(unsigned Opc, unsigned Idx, const APInt &Imm, @@ -1202,6 +1312,12 @@ public: unsigned getCacheLineSize() override { return Impl.getCacheLineSize(); } + llvm::Optional<unsigned> getCacheSize(CacheLevel Level) override { + return Impl.getCacheSize(Level); + } + llvm::Optional<unsigned> getCacheAssociativity(CacheLevel Level) override { + return Impl.getCacheAssociativity(Level); + } unsigned getPrefetchDistance() override { return Impl.getPrefetchDistance(); } unsigned getMinPrefetchStride() override { return Impl.getMinPrefetchStride(); @@ -1267,10 +1383,14 @@ public: return Impl.getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, Alignment, AddressSpace); } - int getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) override { - return Impl.getReductionCost(Opcode, Ty, IsPairwiseForm); + int getArithmeticReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) override { + return Impl.getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm); } + int getMinMaxReductionCost(Type *Ty, Type *CondTy, + bool IsPairwiseForm, bool IsUnsigned) override { + return Impl.getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned); + } int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, ArrayRef<Type *> Tys, FastMathFlags FMF, unsigned ScalarizationCostPassed) override { return Impl.getIntrinsicInstrCost(ID, RetTy, Tys, FMF, @@ -1360,6 +1480,9 @@ public: bool shouldExpandReduction(const IntrinsicInst *II) const override { return Impl.shouldExpandReduction(II); } + int getInstructionLatency(const Instruction *I) override { + return Impl.getInstructionLatency(I); + } }; template <typename T> diff --git a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 9bbda718acab..4c37402278ef 100644 --- a/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/contrib/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -152,6 +152,7 @@ public: case Intrinsic::annotation: case Intrinsic::assume: + case Intrinsic::sideeffect: case Intrinsic::dbg_declare: case Intrinsic::dbg_value: case Intrinsic::invariant_start: @@ -188,6 +189,8 @@ public: } bool isLoweredToCall(const Function *F) { + assert(F && "A concrete function must be provided to this routine."); + // FIXME: These should almost certainly not be handled here, and instead // handled with the help of TLI or the target itself. This was largely // ported from existing analysis heuristics here so that such refactorings @@ -230,7 +233,7 @@ public: bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, - unsigned AddrSpace) { + unsigned AddrSpace, Instruction *I = nullptr) { // Guess that only reg and reg+reg addressing is allowed. This heuristic is // taken from the implementation of LSR. return !BaseGV && BaseOffset == 0 && (Scale == 0 || Scale == 1); @@ -251,6 +254,10 @@ public: bool isLegalMaskedGather(Type *DataType) { return false; } + bool hasDivRemOp(Type *DataType, bool IsSigned) { return false; } + + bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) { return false; } + bool prefersVectorizedAddressing() { return true; } int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, @@ -262,7 +269,7 @@ public: return -1; } - bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) { return true; } + bool LSRWithInstrQueries() { return false; } bool isTruncateFree(Type *Ty1, Type *Ty2) { return false; } @@ -288,7 +295,10 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) { return false; } - bool expandMemCmp(Instruction *I, unsigned &MaxLoadSize) { return false; } + const TTI::MemCmpExpansionOptions *enableMemCmpExpansion( + bool IsZeroCmp) const { + return nullptr; + } bool enableInterleavedAccessVectorization() { return false; } @@ -306,6 +316,8 @@ public: bool haveFastSqrt(Type *Ty) { return false; } + bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) { return true; } + unsigned getFPOpCost(Type *Ty) { return TargetTransformInfo::TCC_Basic; } int getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx, const APInt &Imm, @@ -340,6 +352,29 @@ public: unsigned getCacheLineSize() { return 0; } + llvm::Optional<unsigned> getCacheSize(TargetTransformInfo::CacheLevel Level) { + switch (Level) { + case TargetTransformInfo::CacheLevel::L1D: + LLVM_FALLTHROUGH; + case TargetTransformInfo::CacheLevel::L2D: + return llvm::Optional<unsigned>(); + } + + llvm_unreachable("Unknown TargetTransformInfo::CacheLevel"); + } + + llvm::Optional<unsigned> getCacheAssociativity( + TargetTransformInfo::CacheLevel Level) { + switch (Level) { + case TargetTransformInfo::CacheLevel::L1D: + LLVM_FALLTHROUGH; + case TargetTransformInfo::CacheLevel::L2D: + return llvm::Optional<unsigned>(); + } + + llvm_unreachable("Unknown TargetTransformInfo::CacheLevel"); + } + unsigned getPrefetchDistance() { return 0; } unsigned getMinPrefetchStride() { return 1; } @@ -423,10 +458,12 @@ public: unsigned getAddressComputationCost(Type *Tp, ScalarEvolution *, const SCEV *) { - return 0; + return 0; } - unsigned getReductionCost(unsigned, Type *, bool) { return 1; } + unsigned getArithmeticReductionCost(unsigned, Type *, bool) { return 1; } + + unsigned getMinMaxReductionCost(Type *, Type *, bool, bool) { return 1; } unsigned getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) { return 0; } @@ -587,7 +624,7 @@ protected: APInt StrideVal = Step->getAPInt(); if (StrideVal.getBitWidth() > 64) return false; - // FIXME: need to take absolute value for negtive stride case + // FIXME: Need to take absolute value for negative stride case. return StrideVal.getSExtValue() < MergeDistance; } }; @@ -647,11 +684,13 @@ public: BaseGV = dyn_cast<GlobalValue>(Ptr->stripPointerCasts()); } bool HasBaseReg = (BaseGV == nullptr); - int64_t BaseOffset = 0; + + auto PtrSizeBits = DL.getPointerTypeSizeInBits(Ptr->getType()); + APInt BaseOffset(PtrSizeBits, 0); int64_t Scale = 0; auto GTI = gep_type_begin(PointeeType, Operands); - Type *TargetType; + Type *TargetType = nullptr; // Handle the case where the GEP instruction has a single operand, // the basis, therefore TargetType is a nullptr. @@ -673,9 +712,10 @@ public: BaseOffset += DL.getStructLayout(STy)->getElementOffset(Field); } else { int64_t ElementSize = DL.getTypeAllocSize(GTI.getIndexedType()); - if (ConstIdx) - BaseOffset += ConstIdx->getSExtValue() * ElementSize; - else { + if (ConstIdx) { + BaseOffset += + ConstIdx->getValue().sextOrTrunc(PtrSizeBits) * ElementSize; + } else { // Needs scale register. if (Scale != 0) // No addressing mode takes two scale registers. @@ -688,9 +728,10 @@ public: // Assumes the address space is 0 when Ptr is nullptr. unsigned AS = (Ptr == nullptr ? 0 : Ptr->getType()->getPointerAddressSpace()); + if (static_cast<T *>(this)->isLegalAddressingMode( - TargetType, const_cast<GlobalValue *>(BaseGV), BaseOffset, - HasBaseReg, Scale, AS)) + TargetType, const_cast<GlobalValue *>(BaseGV), + BaseOffset.sextOrTrunc(64).getSExtValue(), HasBaseReg, Scale, AS)) return TTI::TCC_Free; return TTI::TCC_Basic; } @@ -713,6 +754,11 @@ public: if (isa<PHINode>(U)) return TTI::TCC_Free; // Model all PHI nodes as free. + // Static alloca doesn't generate target instructions. + if (auto *A = dyn_cast<AllocaInst>(U)) + if (A->isStaticAlloca()) + return TTI::TCC_Free; + if (const GEPOperator *GEP = dyn_cast<GEPOperator>(U)) { return static_cast<T *>(this)->getGEPCost(GEP->getSourceElementType(), GEP->getPointerOperand(), @@ -746,6 +792,38 @@ public: Operator::getOpcode(U), U->getType(), U->getNumOperands() == 1 ? U->getOperand(0)->getType() : nullptr); } + + int getInstructionLatency(const Instruction *I) { + SmallVector<const Value *, 4> Operands(I->value_op_begin(), + I->value_op_end()); + if (getUserCost(I, Operands) == TTI::TCC_Free) + return 0; + + if (isa<LoadInst>(I)) + return 4; + + Type *DstTy = I->getType(); + + // Usually an intrinsic is a simple instruction. + // A real function call is much slower. + if (auto *CI = dyn_cast<CallInst>(I)) { + const Function *F = CI->getCalledFunction(); + if (!F || static_cast<T *>(this)->isLoweredToCall(F)) + return 40; + // Some intrinsics return a value and a flag, we use the value type + // to decide its latency. + if (StructType* StructTy = dyn_cast<StructType>(DstTy)) + DstTy = StructTy->getElementType(0); + // Fall through to simple instructions. + } + + if (VectorType *VectorTy = dyn_cast<VectorType>(DstTy)) + DstTy = VectorTy->getElementType(); + if (DstTy->isFloatingPointTy()) + return 3; + + return 1; + } }; } diff --git a/contrib/llvm/include/llvm/Analysis/Trace.h b/contrib/llvm/include/llvm/Analysis/Trace.h index bedd654c6521..b05d384ab1a3 100644 --- a/contrib/llvm/include/llvm/Analysis/Trace.h +++ b/contrib/llvm/include/llvm/Analysis/Trace.h @@ -22,39 +22,36 @@ #include <vector> namespace llvm { - class BasicBlock; - class Function; - class Module; - class raw_ostream; + +class BasicBlock; +class Function; +class Module; +class raw_ostream; class Trace { - typedef std::vector<BasicBlock *> BasicBlockListType; + using BasicBlockListType = std::vector<BasicBlock *>; + BasicBlockListType BasicBlocks; public: /// Trace ctor - Make a new trace from a vector of basic blocks, /// residing in the function which is the parent of the first /// basic block in the vector. - /// Trace(const std::vector<BasicBlock *> &vBB) : BasicBlocks (vBB) {} /// getEntryBasicBlock - Return the entry basic block (first block) /// of the trace. - /// BasicBlock *getEntryBasicBlock () const { return BasicBlocks[0]; } /// operator[]/getBlock - Return basic block N in the trace. - /// BasicBlock *operator[](unsigned i) const { return BasicBlocks[i]; } BasicBlock *getBlock(unsigned i) const { return BasicBlocks[i]; } /// getFunction - Return this trace's parent function. - /// Function *getFunction () const; /// getModule - Return this Module that contains this trace's parent /// function. - /// Module *getModule () const; /// getBlockIndex - Return the index of the specified basic block in the @@ -68,14 +65,12 @@ public: /// contains - Returns true if this trace contains the given basic /// block. - /// bool contains(const BasicBlock *X) const { return getBlockIndex(X) != -1; } /// Returns true if B1 occurs before B2 in the trace, or if it is the same /// block as B2.. Both blocks must be in the trace. - /// bool dominates(const BasicBlock *B1, const BasicBlock *B2) const { int B1Idx = getBlockIndex(B1), B2Idx = getBlockIndex(B2); assert(B1Idx != -1 && B2Idx != -1 && "Block is not in the trace!"); @@ -83,10 +78,10 @@ public: } // BasicBlock iterators... - typedef BasicBlockListType::iterator iterator; - typedef BasicBlockListType::const_iterator const_iterator; - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; + using iterator = BasicBlockListType::iterator; + using const_iterator = BasicBlockListType::const_iterator; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; iterator begin() { return BasicBlocks.begin(); } const_iterator begin() const { return BasicBlocks.begin(); } @@ -105,12 +100,10 @@ public: iterator erase(iterator q1, iterator q2) { return BasicBlocks.erase (q1, q2); } /// print - Write trace to output stream. - /// void print(raw_ostream &O) const; /// dump - Debugger convenience method; writes trace to standard error /// output stream. - /// void dump() const; }; diff --git a/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h index fd726e6cd37f..7fcfdb3a817c 100644 --- a/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h +++ b/contrib/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h @@ -6,22 +6,28 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This is the interface for a metadata-based TBAA. See the source file for /// details on the algorithm. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H #define LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Metadata.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include <memory> namespace llvm { +class Function; +class MDNode; +class MemoryLocation; + /// A simple AA result that uses TBAA metadata to answer queries. class TypeBasedAAResult : public AAResultBase<TypeBasedAAResult> { friend AAResultBase<TypeBasedAAResult>; @@ -50,10 +56,11 @@ private: /// Analysis pass providing a never-invalidated alias analysis result. class TypeBasedAA : public AnalysisInfoMixin<TypeBasedAA> { friend AnalysisInfoMixin<TypeBasedAA>; + static AnalysisKey Key; public: - typedef TypeBasedAAResult Result; + using Result = TypeBasedAAResult; TypeBasedAAResult run(Function &F, FunctionAnalysisManager &AM); }; @@ -81,6 +88,7 @@ public: // type-based alias analysis. // ImmutablePass *createTypeBasedAAWrapperPass(); -} -#endif +} // end namespace llvm + +#endif // LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H diff --git a/contrib/llvm/include/llvm/Analysis/ValueLattice.h b/contrib/llvm/include/llvm/Analysis/ValueLattice.h new file mode 100644 index 000000000000..18a43aafa8ca --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/ValueLattice.h @@ -0,0 +1,250 @@ +//===- ValueLattice.h - Value constraint analysis ---------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_VALUELATTICE_H +#define LLVM_ANALYSIS_VALUELATTICE_H + +#include "llvm/IR/ConstantRange.h" +#include "llvm/IR/Constants.h" +// +//===----------------------------------------------------------------------===// +// ValueLatticeElement +//===----------------------------------------------------------------------===// + +/// This class represents lattice values for constants. +/// +/// FIXME: This is basically just for bringup, this can be made a lot more rich +/// in the future. +/// + +namespace llvm { +class ValueLatticeElement { + enum ValueLatticeElementTy { + /// This Value has no known value yet. As a result, this implies the + /// producing instruction is dead. Caution: We use this as the starting + /// state in our local meet rules. In this usage, it's taken to mean + /// "nothing known yet". + undefined, + + /// This Value has a specific constant value. (For constant integers, + /// constantrange is used instead. Integer typed constantexprs can appear + /// as constant.) + constant, + + /// This Value is known to not have the specified value. (For constant + /// integers, constantrange is used instead. As above, integer typed + /// constantexprs can appear here.) + notconstant, + + /// The Value falls within this range. (Used only for integer typed values.) + constantrange, + + /// We can not precisely model the dynamic values this value might take. + overdefined + }; + + /// Val: This stores the current lattice value along with the Constant* for + /// the constant if this is a 'constant' or 'notconstant' value. + ValueLatticeElementTy Tag; + Constant *Val; + ConstantRange Range; + +public: + ValueLatticeElement() : Tag(undefined), Val(nullptr), Range(1, true) {} + + static ValueLatticeElement get(Constant *C) { + ValueLatticeElement Res; + if (!isa<UndefValue>(C)) + Res.markConstant(C); + return Res; + } + static ValueLatticeElement getNot(Constant *C) { + ValueLatticeElement Res; + if (!isa<UndefValue>(C)) + Res.markNotConstant(C); + return Res; + } + static ValueLatticeElement getRange(ConstantRange CR) { + ValueLatticeElement Res; + Res.markConstantRange(std::move(CR)); + return Res; + } + static ValueLatticeElement getOverdefined() { + ValueLatticeElement Res; + Res.markOverdefined(); + return Res; + } + + bool isUndefined() const { return Tag == undefined; } + bool isConstant() const { return Tag == constant; } + bool isNotConstant() const { return Tag == notconstant; } + bool isConstantRange() const { return Tag == constantrange; } + bool isOverdefined() const { return Tag == overdefined; } + + Constant *getConstant() const { + assert(isConstant() && "Cannot get the constant of a non-constant!"); + return Val; + } + + Constant *getNotConstant() const { + assert(isNotConstant() && "Cannot get the constant of a non-notconstant!"); + return Val; + } + + const ConstantRange &getConstantRange() const { + assert(isConstantRange() && + "Cannot get the constant-range of a non-constant-range!"); + return Range; + } + + Optional<APInt> asConstantInteger() const { + if (isConstant() && isa<ConstantInt>(Val)) { + return cast<ConstantInt>(Val)->getValue(); + } else if (isConstantRange() && Range.isSingleElement()) { + return *Range.getSingleElement(); + } + return None; + } + +private: + void markOverdefined() { + if (isOverdefined()) + return; + Tag = overdefined; + } + + void markConstant(Constant *V) { + assert(V && "Marking constant with NULL"); + if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + markConstantRange(ConstantRange(CI->getValue())); + return; + } + if (isa<UndefValue>(V)) + return; + + assert((!isConstant() || getConstant() == V) && + "Marking constant with different value"); + assert(isUndefined()); + Tag = constant; + Val = V; + } + + void markNotConstant(Constant *V) { + assert(V && "Marking constant with NULL"); + if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + markConstantRange(ConstantRange(CI->getValue() + 1, CI->getValue())); + return; + } + if (isa<UndefValue>(V)) + return; + + assert((!isConstant() || getConstant() != V) && + "Marking constant !constant with same value"); + assert((!isNotConstant() || getNotConstant() == V) && + "Marking !constant with different value"); + assert(isUndefined() || isConstant()); + Tag = notconstant; + Val = V; + } + + void markConstantRange(ConstantRange NewR) { + if (isConstantRange()) { + if (NewR.isEmptySet()) + markOverdefined(); + else { + Range = std::move(NewR); + } + return; + } + + assert(isUndefined()); + if (NewR.isEmptySet()) + markOverdefined(); + else { + Tag = constantrange; + Range = std::move(NewR); + } + } + +public: + /// Updates this object to approximate both this object and RHS. Returns + /// true if this object has been changed. + bool mergeIn(const ValueLatticeElement &RHS, const DataLayout &DL) { + if (RHS.isUndefined() || isOverdefined()) + return false; + if (RHS.isOverdefined()) { + markOverdefined(); + return true; + } + + if (isUndefined()) { + *this = RHS; + return !RHS.isUndefined(); + } + + if (isConstant()) { + if (RHS.isConstant() && Val == RHS.Val) + return false; + markOverdefined(); + return true; + } + + if (isNotConstant()) { + if (RHS.isNotConstant() && Val == RHS.Val) + return false; + markOverdefined(); + return true; + } + + assert(isConstantRange() && "New ValueLattice type?"); + if (!RHS.isConstantRange()) { + // We can get here if we've encountered a constantexpr of integer type + // and merge it with a constantrange. + markOverdefined(); + return true; + } + ConstantRange NewR = Range.unionWith(RHS.getConstantRange()); + if (NewR.isFullSet()) + markOverdefined(); + else + markConstantRange(std::move(NewR)); + return true; + } + + ConstantInt *getConstantInt() const { + assert(isConstant() && isa<ConstantInt>(getConstant()) && + "No integer constant"); + return cast<ConstantInt>(getConstant()); + } + + bool satisfiesPredicate(CmpInst::Predicate Pred, + const ValueLatticeElement &Other) const { + // TODO: share with LVI getPredicateResult. + + if (isUndefined() || Other.isUndefined()) + return true; + + if (isConstant() && Other.isConstant() && Pred == CmpInst::FCMP_OEQ) + return getConstant() == Other.getConstant(); + + // Integer constants are represented as ConstantRanges with single + // elements. + if (!isConstantRange() || !Other.isConstantRange()) + return false; + + const auto &CR = getConstantRange(); + const auto &OtherCR = Other.getConstantRange(); + return ConstantRange::makeSatisfyingICmpRegion(Pred, OtherCR).contains(CR); + } +}; + +raw_ostream &operator<<(raw_ostream &OS, const ValueLatticeElement &Val); + +} // end namespace llvm +#endif diff --git a/contrib/llvm/include/llvm/Analysis/ValueLatticeUtils.h b/contrib/llvm/include/llvm/Analysis/ValueLatticeUtils.h new file mode 100644 index 000000000000..02072672e56e --- /dev/null +++ b/contrib/llvm/include/llvm/Analysis/ValueLatticeUtils.h @@ -0,0 +1,41 @@ +//===-- ValueLatticeUtils.h - Utils for solving lattices --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares common functions useful for performing data-flow analyses +// that propagate values across function boundaries. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_VALUELATTICEUTILS_H +#define LLVM_ANALYSIS_VALUELATTICEUTILS_H + +namespace llvm { + +class Function; +class GlobalVariable; + +/// Determine if the values of the given function's arguments can be tracked +/// interprocedurally. The value of an argument can be tracked if the function +/// has local linkage and its address is not taken. +bool canTrackArgumentsInterprocedurally(Function *F); + +/// Determine if the values of the given function's returns can be tracked +/// interprocedurally. Return values can be tracked if the function has an +/// exact definition and it doesn't have the "naked" attribute. Naked functions +/// may contain assembly code that returns untrackable values. +bool canTrackReturnsInterprocedurally(Function *F); + +/// Determine if the value maintained in the given global variable can be +/// tracked interprocedurally. A value can be tracked if the global variable +/// has local linkage and is only used by non-volatile loads and stores. +bool canTrackGlobalVariableInterprocedurally(GlobalVariable *GV); + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_VALUELATTICEUTILS_H diff --git a/contrib/llvm/include/llvm/Analysis/ValueTracking.h b/contrib/llvm/include/llvm/Analysis/ValueTracking.h index da058b1d3918..1c51523b1573 100644 --- a/contrib/llvm/include/llvm/Analysis/ValueTracking.h +++ b/contrib/llvm/include/llvm/Analysis/ValueTracking.h @@ -15,32 +15,32 @@ #ifndef LLVM_ANALYSIS_VALUETRACKING_H #define LLVM_ANALYSIS_VALUETRACKING_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Instruction.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/Support/DataTypes.h" +#include "llvm/IR/Intrinsics.h" +#include <cassert> +#include <cstdint> namespace llvm { -template <typename T> class ArrayRef; - class APInt; - class AddOperator; - class AssumptionCache; - class DataLayout; - class DominatorTree; - class GEPOperator; - class Instruction; - struct KnownBits; - class Loop; - class LoopInfo; - class OptimizationRemarkEmitter; - class MDNode; - class StringRef; - class TargetLibraryInfo; - class Value; - - namespace Intrinsic { - enum ID : unsigned; - } + +class AddOperator; +class APInt; +class AssumptionCache; +class DataLayout; +class DominatorTree; +class GEPOperator; +class IntrinsicInst; +struct KnownBits; +class Loop; +class LoopInfo; +class MDNode; +class OptimizationRemarkEmitter; +class StringRef; +class TargetLibraryInfo; +class Value; /// Determine which bits of V are known to be either zero or one and return /// them in the KnownZero/KnownOne bit sets. @@ -56,17 +56,20 @@ template <typename T> class ArrayRef; const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr, OptimizationRemarkEmitter *ORE = nullptr); + /// Returns the known bits rather than passing by reference. KnownBits computeKnownBits(const Value *V, const DataLayout &DL, unsigned Depth = 0, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr, OptimizationRemarkEmitter *ORE = nullptr); + /// Compute known bits from the range metadata. /// \p KnownZero the set of bits that are known to be zero /// \p KnownOne the set of bits that are known to be one void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known); + /// Return true if LHS and RHS have no common bits set. bool haveNoCommonBitsSet(const Value *LHS, const Value *RHS, const DataLayout &DL, @@ -86,7 +89,7 @@ template <typename T> class ArrayRef; const DominatorTree *DT = nullptr); bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI); - + /// Return true if the given value is known to be non-zero when defined. For /// vectors, return true if every element is known to be non-zero when /// defined. For pointers, if the context instruction and dominator tree are @@ -180,9 +183,13 @@ template <typename T> class ArrayRef; /// -0 --> true /// x > +0 --> true /// x < -0 --> false - /// bool CannotBeOrderedLessThanZero(const Value *V, const TargetLibraryInfo *TLI); + /// Return true if the floating-point scalar value is not a NaN or if the + /// floating-point vector value has no NaN elements. Return false if a value + /// could ever be NaN. + bool isKnownNeverNaN(const Value *V); + /// Return true if we can prove that the specified FP value's sign bit is 0. /// /// NaN --> true/false (depending on the NaN's sign bit) @@ -190,7 +197,6 @@ template <typename T> class ArrayRef; /// -0 --> false /// x > +0 --> true /// x < -0 --> false - /// bool SignBitMustBeZero(const Value *V, const TargetLibraryInfo *TLI); /// If the specified value can be set by repeating the same byte in memory, @@ -214,9 +220,9 @@ template <typename T> class ArrayRef; /// pointer plus a constant offset. Return the base and offset to the caller. Value *GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL); - static inline const Value * - GetPointerBaseWithConstantOffset(const Value *Ptr, int64_t &Offset, - const DataLayout &DL) { + inline const Value *GetPointerBaseWithConstantOffset(const Value *Ptr, + int64_t &Offset, + const DataLayout &DL) { return GetPointerBaseWithConstantOffset(const_cast<Value *>(Ptr), Offset, DL); } @@ -231,8 +237,10 @@ template <typename T> class ArrayRef; /// ConstantDataArray pointer. nullptr indicates a zeroinitializer (a valid /// initializer, it just doesn't fit the ConstantDataArray interface). const ConstantDataArray *Array; + /// Slice starts at this Offset. uint64_t Offset; + /// Length of the slice. uint64_t Length; @@ -242,14 +250,15 @@ template <typename T> class ArrayRef; Offset += Delta; Length -= Delta; } + /// Convenience accessor for elements in the slice. uint64_t operator[](unsigned I) const { return Array==nullptr ? 0 : Array->getElementAsInteger(I + Offset); } }; - /// Returns true if the value \p V is a pointer into a ContantDataArray. - /// If successful \p Index will point to a ConstantDataArray info object + /// Returns true if the value \p V is a pointer into a ConstantDataArray. + /// If successful \p Slice will point to a ConstantDataArray info object /// with an appropriate offset. bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset = 0); @@ -274,9 +283,8 @@ template <typename T> class ArrayRef; /// be stripped off. Value *GetUnderlyingObject(Value *V, const DataLayout &DL, unsigned MaxLookup = 6); - static inline const Value *GetUnderlyingObject(const Value *V, - const DataLayout &DL, - unsigned MaxLookup = 6) { + inline const Value *GetUnderlyingObject(const Value *V, const DataLayout &DL, + unsigned MaxLookup = 6) { return GetUnderlyingObject(const_cast<Value *>(V), DL, MaxLookup); } @@ -314,7 +322,7 @@ template <typename T> class ArrayRef; /// This is a wrapper around GetUnderlyingObjects and adds support for basic /// ptrtoint+arithmetic+inttoptr sequences. - void getUnderlyingObjectsForCodeGen(const Value *V, + bool getUnderlyingObjectsForCodeGen(const Value *V, SmallVectorImpl<Value *> &Objects, const DataLayout &DL); @@ -358,18 +366,9 @@ template <typename T> class ArrayRef; /// operands are not memory dependent. bool mayBeMemoryDependent(const Instruction &I); - /// Return true if this pointer couldn't possibly be null by its definition. - /// This returns true for allocas, non-extern-weak globals, and byval - /// arguments. - bool isKnownNonNull(const Value *V); - - /// Return true if this pointer couldn't possibly be null. If the context - /// instruction and dominator tree are specified, perform context-sensitive - /// analysis and return true if the pointer couldn't possibly be null at the - /// specified instruction. - bool isKnownNonNullAt(const Value *V, - const Instruction *CtxI = nullptr, - const DominatorTree *DT = nullptr); + /// Return true if it is an intrinsic that cannot be speculated but also + /// cannot trap. + bool isAssumeLikeIntrinsic(const Instruction *I); /// Return true if it is valid to use the assumptions provided by an /// assume intrinsic, I, at the point in the control-flow identified by the @@ -378,6 +377,7 @@ template <typename T> class ArrayRef; const DominatorTree *DT = nullptr); enum class OverflowResult { AlwaysOverflows, MayOverflow, NeverOverflows }; + OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, const DataLayout &DL, @@ -466,6 +466,7 @@ template <typename T> class ArrayRef; SPF_ABS, /// Absolute value SPF_NABS /// Negated absolute value }; + /// \brief Behavior when a floating point min/max is given one NaN and one /// non-NaN as input. enum SelectPatternNaNBehavior { @@ -476,6 +477,7 @@ template <typename T> class ArrayRef; /// it has been determined that no operands can /// be NaN). }; + struct SelectPatternResult { SelectPatternFlavor Flavor; SelectPatternNaNBehavior NaNBehavior; /// Only applicable if Flavor is @@ -489,6 +491,7 @@ template <typename T> class ArrayRef; return !(SPF == SPF_UNKNOWN || SPF == SPF_ABS || SPF == SPF_NABS); } }; + /// Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind /// and providing the out parameter results if we successfully match. /// @@ -506,7 +509,7 @@ template <typename T> class ArrayRef; /// SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp = nullptr); - static inline SelectPatternResult + inline SelectPatternResult matchSelectPattern(const Value *V, const Value *&LHS, const Value *&RHS, Instruction::CastOps *CastOp = nullptr) { Value *L = const_cast<Value*>(LHS); @@ -528,11 +531,8 @@ template <typename T> class ArrayRef; /// F | T | T /// (A) Optional<bool> isImpliedCondition(const Value *LHS, const Value *RHS, - const DataLayout &DL, - bool LHSIsFalse = false, unsigned Depth = 0, - AssumptionCache *AC = nullptr, - const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); + const DataLayout &DL, bool LHSIsTrue = true, + unsigned Depth = 0); } // end namespace llvm -#endif +#endif // LLVM_ANALYSIS_VALUETRACKING_H diff --git a/contrib/llvm/include/llvm/AsmParser/Parser.h b/contrib/llvm/include/llvm/AsmParser/Parser.h index 768b089b8a2a..5f02e488e5b1 100644 --- a/contrib/llvm/include/llvm/AsmParser/Parser.h +++ b/contrib/llvm/include/llvm/AsmParser/Parser.h @@ -36,10 +36,12 @@ class Type; /// \param Context Context in which to allocate globals info. /// \param Slots The optional slot mapping that will be initialized during /// parsing. -std::unique_ptr<Module> parseAssemblyFile(StringRef Filename, - SMDiagnostic &Error, - LLVMContext &Context, - SlotMapping *Slots = nullptr); +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! +std::unique_ptr<Module> +parseAssemblyFile(StringRef Filename, SMDiagnostic &Error, LLVMContext &Context, + SlotMapping *Slots = nullptr, bool UpgradeDebugInfo = true); /// The function is a secondary interface to the LLVM Assembly Parser. It parses /// an ASCII string that (presumably) contains LLVM Assembly code. It returns a @@ -52,10 +54,14 @@ std::unique_ptr<Module> parseAssemblyFile(StringRef Filename, /// \param Context Context in which to allocate globals info. /// \param Slots The optional slot mapping that will be initialized during /// parsing. +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! std::unique_ptr<Module> parseAssemblyString(StringRef AsmString, SMDiagnostic &Error, LLVMContext &Context, - SlotMapping *Slots = nullptr); + SlotMapping *Slots = nullptr, + bool UpgradeDebugInfo = true); /// parseAssemblyFile and parseAssemblyString are wrappers around this function. /// \brief Parse LLVM Assembly from a MemoryBuffer. @@ -63,9 +69,13 @@ std::unique_ptr<Module> parseAssemblyString(StringRef AsmString, /// \param Err Error result info. /// \param Slots The optional slot mapping that will be initialized during /// parsing. +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! std::unique_ptr<Module> parseAssembly(MemoryBufferRef F, SMDiagnostic &Err, LLVMContext &Context, - SlotMapping *Slots = nullptr); + SlotMapping *Slots = nullptr, + bool UpgradeDebugInfo = true); /// This function is the low-level interface to the LLVM Assembly Parser. /// This is kept as an independent function instead of being inlined into @@ -78,8 +88,12 @@ std::unique_ptr<Module> parseAssembly(MemoryBufferRef F, SMDiagnostic &Err, /// \param Slots The optional slot mapping that will be initialized during /// parsing. /// \return true on error. +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! bool parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err, - SlotMapping *Slots = nullptr); + SlotMapping *Slots = nullptr, + bool UpgradeDebugInfo = true); /// Parse a type and a constant value in the given string. /// diff --git a/contrib/llvm/include/llvm/BinaryFormat/COFF.h b/contrib/llvm/include/llvm/BinaryFormat/COFF.h index b395db6eaa83..e6bb50cadb12 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/COFF.h +++ b/contrib/llvm/include/llvm/BinaryFormat/COFF.h @@ -91,7 +91,7 @@ struct BigObjHeader { uint32_t NumberOfSymbols; }; -enum MachineTypes { +enum MachineTypes : unsigned { MT_Invalid = 0xffff, IMAGE_FILE_MACHINE_UNKNOWN = 0x0, @@ -118,7 +118,7 @@ enum MachineTypes { IMAGE_FILE_MACHINE_WCEMIPSV2 = 0x169 }; -enum Characteristics { +enum Characteristics : unsigned { C_Invalid = 0, /// The file does not contain base relocations and must be loaded at its @@ -158,7 +158,7 @@ enum Characteristics { IMAGE_FILE_BYTES_REVERSED_HI = 0x8000 }; -enum ResourceTypeID { +enum ResourceTypeID : unsigned { RID_Cursor = 1, RID_Bitmap = 2, RID_Icon = 3, @@ -234,7 +234,7 @@ enum SymbolStorageClass { IMAGE_SYM_CLASS_CLR_TOKEN = 107 }; -enum SymbolBaseType { +enum SymbolBaseType : unsigned { IMAGE_SYM_TYPE_NULL = 0, ///< No type information or unknown base type. IMAGE_SYM_TYPE_VOID = 1, ///< Used with void pointers and functions. IMAGE_SYM_TYPE_CHAR = 2, ///< A character (signed byte). @@ -253,7 +253,7 @@ enum SymbolBaseType { IMAGE_SYM_TYPE_DWORD = 15 ///< An unsigned 4-byte integer. }; -enum SymbolComplexType { +enum SymbolComplexType : unsigned { IMAGE_SYM_DTYPE_NULL = 0, ///< No complex type; simple scalar variable. IMAGE_SYM_DTYPE_POINTER = 1, ///< A pointer to base type. IMAGE_SYM_DTYPE_FUNCTION = 2, ///< A function that returns a base type. @@ -325,7 +325,7 @@ struct relocation { uint16_t Type; }; -enum RelocationTypeI386 { +enum RelocationTypeI386 : unsigned { IMAGE_REL_I386_ABSOLUTE = 0x0000, IMAGE_REL_I386_DIR16 = 0x0001, IMAGE_REL_I386_REL16 = 0x0002, @@ -339,7 +339,7 @@ enum RelocationTypeI386 { IMAGE_REL_I386_REL32 = 0x0014 }; -enum RelocationTypeAMD64 { +enum RelocationTypeAMD64 : unsigned { IMAGE_REL_AMD64_ABSOLUTE = 0x0000, IMAGE_REL_AMD64_ADDR64 = 0x0001, IMAGE_REL_AMD64_ADDR32 = 0x0002, @@ -359,7 +359,7 @@ enum RelocationTypeAMD64 { IMAGE_REL_AMD64_SSPAN32 = 0x0010 }; -enum RelocationTypesARM { +enum RelocationTypesARM : unsigned { IMAGE_REL_ARM_ABSOLUTE = 0x0000, IMAGE_REL_ARM_ADDR32 = 0x0001, IMAGE_REL_ARM_ADDR32NB = 0x0002, @@ -377,7 +377,7 @@ enum RelocationTypesARM { IMAGE_REL_ARM_BLX23T = 0x0015 }; -enum RelocationTypesARM64 { +enum RelocationTypesARM64 : unsigned { IMAGE_REL_ARM64_ABSOLUTE = 0x0000, IMAGE_REL_ARM64_ADDR32 = 0x0001, IMAGE_REL_ARM64_ADDR32NB = 0x0002, @@ -397,7 +397,7 @@ enum RelocationTypesARM64 { IMAGE_REL_ARM64_BRANCH14 = 0x0010, }; -enum COMDATType { +enum COMDATType : unsigned { IMAGE_COMDAT_SELECT_NODUPLICATES = 1, IMAGE_COMDAT_SELECT_ANY, IMAGE_COMDAT_SELECT_SAME_SIZE, @@ -430,7 +430,7 @@ struct AuxiliaryWeakExternal { uint8_t unused[10]; }; -enum WeakExternalCharacteristics { +enum WeakExternalCharacteristics : unsigned { IMAGE_WEAK_EXTERN_SEARCH_NOLIBRARY = 1, IMAGE_WEAK_EXTERN_SEARCH_LIBRARY = 2, IMAGE_WEAK_EXTERN_SEARCH_ALIAS = 3 @@ -572,7 +572,7 @@ struct DataDirectory { uint32_t Size; }; -enum DataDirectoryIndex { +enum DataDirectoryIndex : unsigned { EXPORT_TABLE = 0, IMPORT_TABLE, RESOURCE_TABLE, @@ -592,7 +592,7 @@ enum DataDirectoryIndex { NUM_DATA_DIRECTORIES }; -enum WindowsSubsystem { +enum WindowsSubsystem : unsigned { IMAGE_SUBSYSTEM_UNKNOWN = 0, ///< An unknown subsystem. IMAGE_SUBSYSTEM_NATIVE = 1, ///< Device drivers and native Windows processes IMAGE_SUBSYSTEM_WINDOWS_GUI = 2, ///< The Windows GUI subsystem. @@ -611,7 +611,7 @@ enum WindowsSubsystem { IMAGE_SUBSYSTEM_WINDOWS_BOOT_APPLICATION = 16 ///< A BCD application. }; -enum DLLCharacteristics { +enum DLLCharacteristics : unsigned { /// ASLR with 64 bit address space. IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020, /// DLL can be relocated at load time. @@ -637,7 +637,7 @@ enum DLLCharacteristics { IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE = 0x8000 }; -enum DebugType { +enum DebugType : unsigned { IMAGE_DEBUG_TYPE_UNKNOWN = 0, IMAGE_DEBUG_TYPE_COFF = 1, IMAGE_DEBUG_TYPE_CODEVIEW = 2, @@ -657,7 +657,7 @@ enum DebugType { IMAGE_DEBUG_TYPE_REPRO = 16, }; -enum BaseRelocationType { +enum BaseRelocationType : unsigned { IMAGE_REL_BASED_ABSOLUTE = 0, IMAGE_REL_BASED_HIGH = 1, IMAGE_REL_BASED_LOW = 2, @@ -670,9 +670,13 @@ enum BaseRelocationType { IMAGE_REL_BASED_DIR64 = 10 }; -enum ImportType { IMPORT_CODE = 0, IMPORT_DATA = 1, IMPORT_CONST = 2 }; +enum ImportType : unsigned { + IMPORT_CODE = 0, + IMPORT_DATA = 1, + IMPORT_CONST = 2 +}; -enum ImportNameType { +enum ImportNameType : unsigned { /// Import is by ordinal. This indicates that the value in the Ordinal/Hint /// field of the import header is the import's ordinal. If this constant is /// not specified, then the Ordinal/Hint field should always be interpreted @@ -707,6 +711,7 @@ struct ImportHeader { enum CodeViewIdentifiers { DEBUG_SECTION_MAGIC = 0x4, + DEBUG_HASHES_SECTION_MAGIC = 0x133C9C5 }; inline bool isReservedSectionNumber(int32_t SectionNumber) { diff --git a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def index 3df3300de466..3ade3ea0d338 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def +++ b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.def @@ -20,7 +20,7 @@ defined HANDLE_DW_LNE || defined HANDLE_DW_LNCT || \ defined HANDLE_DW_MACRO || defined HANDLE_DW_RLE || \ defined HANDLE_DW_CFA || defined HANDLE_DW_APPLE_PROPERTY || \ - defined HANDLE_DW_UT) + defined HANDLE_DW_UT || defined HANDLE_DWARF_SECTION) #error "Missing macro definition of HANDLE_DW*" #endif @@ -92,6 +92,10 @@ #define HANDLE_DW_UT(ID, NAME) #endif +#ifndef HANDLE_DWARF_SECTION +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME) +#endif + HANDLE_DW_TAG(0x0000, null, 2, DWARF) HANDLE_DW_TAG(0x0001, array_type, 2, DWARF) HANDLE_DW_TAG(0x0002, class_type, 2, DWARF) @@ -172,6 +176,8 @@ HANDLE_DW_TAG(0x4103, class_template, 0, GNU) HANDLE_DW_TAG(0x4106, GNU_template_template_param, 0, GNU) HANDLE_DW_TAG(0x4107, GNU_template_parameter_pack, 0, GNU) HANDLE_DW_TAG(0x4108, GNU_formal_parameter_pack, 0, GNU) +HANDLE_DW_TAG(0x4109, GNU_call_site, 0, GNU) +HANDLE_DW_TAG(0x410a, GNU_call_site_parameter, 0, GNU) HANDLE_DW_TAG(0x4200, APPLE_property, 0, APPLE) HANDLE_DW_TAG(0xb000, BORLAND_property, 0, BORLAND) HANDLE_DW_TAG(0xb001, BORLAND_Delphi_string, 0, BORLAND) @@ -334,6 +340,8 @@ HANDLE_DW_AT(0x2106, body_end, 0, GNU) HANDLE_DW_AT(0x2107, GNU_vector, 0, GNU) HANDLE_DW_AT(0x2110, GNU_template_name, 0, GNU) HANDLE_DW_AT(0x210f, GNU_odr_signature, 0, GNU) +HANDLE_DW_AT(0x2111, GNU_call_site_value, 0, GNU) +HANDLE_DW_AT(0x2117, GNU_all_call_sites, 0, GNU) HANDLE_DW_AT(0x2119, GNU_macros, 0, GNU) // Extensions for Fission proposal. HANDLE_DW_AT(0x2130, GNU_dwo_name, 0, GNU) @@ -819,6 +827,35 @@ HANDLE_DW_UT(0x04, skeleton) HANDLE_DW_UT(0x05, split_compile) HANDLE_DW_UT(0x06, split_type) +// DWARF section types. (enum name, ELF name, ELF DWO name, cmdline name) +// Note that these IDs don't mean anything. +// TODO: Add Mach-O and COFF names. +// Official DWARF sections. +HANDLE_DWARF_SECTION(DebugAbbrev, ".debug_abbrev", "debug-abbrev") +HANDLE_DWARF_SECTION(DebugAranges, ".debug_aranges", "debug-aranges") +HANDLE_DWARF_SECTION(DebugInfo, ".debug_info", "debug-info") +HANDLE_DWARF_SECTION(DebugTypes, ".debug_types", "debug-types") +HANDLE_DWARF_SECTION(DebugLine, ".debug_line", "debug-line") +HANDLE_DWARF_SECTION(DebugLoc, ".debug_loc", "debug-loc") +HANDLE_DWARF_SECTION(DebugFrame, ".debug_frame", "debug-frame") +HANDLE_DWARF_SECTION(DebugMacro, ".debug_macro", "debug-macro") +HANDLE_DWARF_SECTION(DebugRanges, ".debug_ranges", "debug-ranges") +HANDLE_DWARF_SECTION(DebugPubnames, ".debug_pubnames", "debug-pubnames") +HANDLE_DWARF_SECTION(DebugPubtypes, ".debug_pubtypes", "debug-pubtypes") +HANDLE_DWARF_SECTION(DebugGnuPubnames, ".debug_gnu_pubnames", "debug-gnu-pubnames") +HANDLE_DWARF_SECTION(DebugGnuPubtypes, ".debug_gnu_pubtypes", "debug-gnu-pubtypes") +HANDLE_DWARF_SECTION(DebugStr, ".debug_str", "debug-str") +HANDLE_DWARF_SECTION(DebugStrOffsets, ".debug_str_offsets", "debug-str-offsets") +HANDLE_DWARF_SECTION(DebugCUIndex, ".debug_cu_index", "debug-cu-index") +HANDLE_DWARF_SECTION(DebugTUIndex, ".debug_tu_index", "debug-tu-index") +// Vendor extensions. +HANDLE_DWARF_SECTION(AppleNames, ".apple_names", "apple-names") +HANDLE_DWARF_SECTION(AppleTypes, ".apple_types", "apple-types") +HANDLE_DWARF_SECTION(AppleNamespaces, ".apple_namespaces", "apple-namespaces") +HANDLE_DWARF_SECTION(AppleObjC, ".apple_objc", "apple-objc") +HANDLE_DWARF_SECTION(GdbIndex, ".gdb_index", "gdb-index") + + #undef HANDLE_DW_TAG #undef HANDLE_DW_AT #undef HANDLE_DW_FORM @@ -836,3 +873,4 @@ HANDLE_DW_UT(0x06, split_type) #undef HANDLE_DW_CFA #undef HANDLE_DW_APPLE_PROPERTY #undef HANDLE_DW_UT +#undef HANDLE_DWARF_SECTION diff --git a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h index 80456a0808f2..a0e5367b412c 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/contrib/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -7,13 +7,13 @@ // //===----------------------------------------------------------------------===// // -// \file -// \brief This file contains constants used for implementing Dwarf -// debug support. -// -// For details on the Dwarf specfication see the latest DWARF Debugging -// Information Format standard document on http://www.dwarfstd.org. This -// file often includes support for non-released standard features. +/// \file +/// This file contains constants used for implementing Dwarf +/// debug support. +/// +/// For details on the Dwarf specfication see the latest DWARF Debugging +/// Information Format standard document on http://www.dwarfstd.org. This +/// file often includes support for non-released standard features. // //===----------------------------------------------------------------------===// @@ -57,12 +57,14 @@ enum LLVMConstants : uint32_t { DWARF_VENDOR_MIPS = 6 }; -// Special ID values that distinguish a CIE from a FDE in DWARF CFI. -// Not inside an enum because a 64-bit value is needed. +/// Special ID values that distinguish a CIE from a FDE in DWARF CFI. +/// Not inside an enum because a 64-bit value is needed. +/// @{ const uint32_t DW_CIE_ID = UINT32_MAX; const uint64_t DW64_CIE_ID = UINT64_MAX; +/// @} -// Identifier of an invalid DIE offset in the .debug_info section. +/// Identifier of an invalid DIE offset in the .debug_info section. const uint32_t DW_INVALID_OFFSET = UINT32_MAX; enum Tag : uint16_t { @@ -70,7 +72,7 @@ enum Tag : uint16_t { #include "llvm/BinaryFormat/Dwarf.def" DW_TAG_lo_user = 0x4080, DW_TAG_hi_user = 0xffff, - DW_TAG_user_base = 0x1000 // Recommended base for user tags. + DW_TAG_user_base = 0x1000 ///< Recommended base for user tags. }; inline bool isType(Tag T) { @@ -323,6 +325,32 @@ enum UnitType : unsigned char { DW_UT_hi_user = 0xff }; +inline bool isUnitType(uint8_t UnitType) { + switch (UnitType) { + case DW_UT_compile: + case DW_UT_type: + case DW_UT_partial: + case DW_UT_skeleton: + case DW_UT_split_compile: + case DW_UT_split_type: + return true; + default: + return false; + } +} + +inline bool isUnitType(dwarf::Tag T) { + switch (T) { + case DW_TAG_compile_unit: + case DW_TAG_type_unit: + case DW_TAG_partial_unit: + case DW_TAG_skeleton_unit: + return true; + default: + return false; + } +} + // Constants for the DWARF v5 Accelerator Table Proposal enum AcceleratorTable { // Data layout descriptors. @@ -447,11 +475,11 @@ unsigned LanguageVendor(SourceLanguage L); /// or is an extension if extensions are allowed. bool isValidFormForVersion(Form F, unsigned Version, bool ExtensionsOk = true); -/// \brief Returns the symbolic string representing Val when used as a value +/// Returns the symbolic string representing Val when used as a value /// for attribute Attr. StringRef AttributeValueString(uint16_t Attr, unsigned Val); -/// \brief Decsribes an entry of the various gnu_pub* debug sections. +/// Describes an entry of the various gnu_pub* debug sections. /// /// The gnu_pub* kind looks like: /// @@ -489,6 +517,9 @@ private: /// Constants that define the DWARF format as 32 or 64 bit. enum DwarfFormat : uint8_t { DWARF32, DWARF64 }; +/// The Bernstein hash function used by the accelerator tables. +uint32_t djbHash(StringRef Buffer); + } // End of namespace dwarf } // End of namespace llvm diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELF.h b/contrib/llvm/include/llvm/BinaryFormat/ELF.h index a4450ee13b40..c902972d93bd 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/ELF.h +++ b/contrib/llvm/include/llvm/BinaryFormat/ELF.h @@ -335,29 +335,33 @@ enum { // OS ABI identification. enum { - ELFOSABI_NONE = 0, // UNIX System V ABI - ELFOSABI_HPUX = 1, // HP-UX operating system - ELFOSABI_NETBSD = 2, // NetBSD - ELFOSABI_GNU = 3, // GNU/Linux - ELFOSABI_LINUX = 3, // Historical alias for ELFOSABI_GNU. - ELFOSABI_HURD = 4, // GNU/Hurd - ELFOSABI_SOLARIS = 6, // Solaris - ELFOSABI_AIX = 7, // AIX - ELFOSABI_IRIX = 8, // IRIX - ELFOSABI_FREEBSD = 9, // FreeBSD - ELFOSABI_TRU64 = 10, // TRU64 UNIX - ELFOSABI_MODESTO = 11, // Novell Modesto - ELFOSABI_OPENBSD = 12, // OpenBSD - ELFOSABI_OPENVMS = 13, // OpenVMS - ELFOSABI_NSK = 14, // Hewlett-Packard Non-Stop Kernel - ELFOSABI_AROS = 15, // AROS - ELFOSABI_FENIXOS = 16, // FenixOS - ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI - ELFOSABI_C6000_ELFABI = 64, // Bare-metal TMS320C6000 - ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime - ELFOSABI_C6000_LINUX = 65, // Linux TMS320C6000 - ELFOSABI_ARM = 97, // ARM - ELFOSABI_STANDALONE = 255 // Standalone (embedded) application + ELFOSABI_NONE = 0, // UNIX System V ABI + ELFOSABI_HPUX = 1, // HP-UX operating system + ELFOSABI_NETBSD = 2, // NetBSD + ELFOSABI_GNU = 3, // GNU/Linux + ELFOSABI_LINUX = 3, // Historical alias for ELFOSABI_GNU. + ELFOSABI_HURD = 4, // GNU/Hurd + ELFOSABI_SOLARIS = 6, // Solaris + ELFOSABI_AIX = 7, // AIX + ELFOSABI_IRIX = 8, // IRIX + ELFOSABI_FREEBSD = 9, // FreeBSD + ELFOSABI_TRU64 = 10, // TRU64 UNIX + ELFOSABI_MODESTO = 11, // Novell Modesto + ELFOSABI_OPENBSD = 12, // OpenBSD + ELFOSABI_OPENVMS = 13, // OpenVMS + ELFOSABI_NSK = 14, // Hewlett-Packard Non-Stop Kernel + ELFOSABI_AROS = 15, // AROS + ELFOSABI_FENIXOS = 16, // FenixOS + ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI + ELFOSABI_FIRST_ARCH = 64, // First architecture-specific OS ABI + ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime + ELFOSABI_AMDGPU_PAL = 65, // AMD PAL runtime + ELFOSABI_AMDGPU_MESA3D = 66, // AMD GCN GPUs (GFX6+) for MESA runtime + ELFOSABI_ARM = 97, // ARM + ELFOSABI_C6000_ELFABI = 64, // Bare-metal TMS320C6000 + ELFOSABI_C6000_LINUX = 65, // Linux TMS320C6000 + ELFOSABI_STANDALONE = 255, // Standalone (embedded) application + ELFOSABI_LAST_ARCH = 255 // Last Architecture-specific OS ABI }; #define ELF_RELOC(name, value) name = value, @@ -430,6 +434,27 @@ enum { #include "ELFRelocs/ARM.def" }; +// ARC Specific e_flags +enum : unsigned { + EF_ARC_MACH_MSK = 0x000000ff, + EF_ARC_OSABI_MSK = 0x00000f00, + E_ARC_MACH_ARC600 = 0x00000002, + E_ARC_MACH_ARC601 = 0x00000004, + E_ARC_MACH_ARC700 = 0x00000003, + EF_ARC_CPU_ARCV2EM = 0x00000005, + EF_ARC_CPU_ARCV2HS = 0x00000006, + E_ARC_OSABI_ORIG = 0x00000000, + E_ARC_OSABI_V2 = 0x00000200, + E_ARC_OSABI_V3 = 0x00000300, + E_ARC_OSABI_V4 = 0x00000400, + EF_ARC_PIC = 0x00000100 +}; + +// ELF Relocation types for ARC +enum { +#include "ELFRelocs/ARC.def" +}; + // AVR specific e_flags enum : unsigned { EF_AVR_ARCH_AVR1 = 1, @@ -559,6 +584,7 @@ enum { EF_HEXAGON_MACH_V55 = 0x00000005, // Hexagon V55 EF_HEXAGON_MACH_V60 = 0x00000060, // Hexagon V60 EF_HEXAGON_MACH_V62 = 0x00000062, // Hexagon V62 + EF_HEXAGON_MACH_V65 = 0x00000065, // Hexagon V65 // Highest ISA version flags EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0] @@ -570,6 +596,7 @@ enum { EF_HEXAGON_ISA_V55 = 0x00000050, // Hexagon V55 ISA EF_HEXAGON_ISA_V60 = 0x00000060, // Hexagon V60 ISA EF_HEXAGON_ISA_V62 = 0x00000062, // Hexagon V62 ISA + EF_HEXAGON_ISA_V65 = 0x00000065, // Hexagon V65 ISA }; // Hexagon-specific section indexes for common small data @@ -591,6 +618,17 @@ enum { #include "ELFRelocs/Lanai.def" }; +// RISCV Specific e_flags +enum : unsigned { + EF_RISCV_RVC = 0x0001, + EF_RISCV_FLOAT_ABI = 0x0006, + EF_RISCV_FLOAT_ABI_SOFT = 0x0000, + EF_RISCV_FLOAT_ABI_SINGLE = 0x0002, + EF_RISCV_FLOAT_ABI_DOUBLE = 0x0004, + EF_RISCV_FLOAT_ABI_QUAD = 0x0006, + EF_RISCV_RVE = 0x0008 +}; + // ELF Relocation types for RISC-V enum { #include "ELFRelocs/RISCV.def" @@ -611,6 +649,15 @@ enum { #include "ELFRelocs/WebAssembly.def" }; +// AMDGPU specific e_flags. +enum : unsigned { + // AMDGPU machine architectures. + EF_AMDGPU_ARCH_NONE = 0x00000000, // None/unknown. + EF_AMDGPU_ARCH_R600 = 0x00000001, // AMD HD2XXX-HD6XXX GPUs. + EF_AMDGPU_ARCH_GCN = 0x00000002, // AMD GCN GFX6+ GPUs. + EF_AMDGPU_ARCH = 0x0000000f // EF_AMDGPU_ARCH_XXX selection mask. +}; + // ELF Relocation types for AMDGPU enum { #include "ELFRelocs/AMDGPU.def" @@ -685,6 +732,10 @@ enum : unsigned { SHT_GROUP = 17, // Section group. SHT_SYMTAB_SHNDX = 18, // Indices for SHN_XINDEX entries. SHT_LOOS = 0x60000000, // Lowest operating system-specific type. + // Android packed relocation section types. + // https://android.googlesource.com/platform/bionic/+/6f12bfece5dcc01325e0abba56a46b1bcf991c69/tools/relocation_packer/src/elf_file.cc#37 + SHT_ANDROID_REL = 0x60000001, + SHT_ANDROID_RELA = 0x60000002, SHT_LLVM_ODRTAB = 0x6fff4c00, // LLVM ODR table. SHT_GNU_ATTRIBUTES = 0x6ffffff5, // Object attributes. SHT_GNU_HASH = 0x6ffffff6, // GNU-style hash table. @@ -1121,6 +1172,13 @@ enum { DT_LOPROC = 0x70000000, // Start of processor specific tags. DT_HIPROC = 0x7FFFFFFF, // End of processor specific tags. + // Android packed relocation section tags. + // https://android.googlesource.com/platform/bionic/+/6f12bfece5dcc01325e0abba56a46b1bcf991c69/tools/relocation_packer/src/elf_file.cc#31 + DT_ANDROID_REL = 0x6000000F, + DT_ANDROID_RELSZ = 0x60000010, + DT_ANDROID_RELA = 0x60000011, + DT_ANDROID_RELASZ = 0x60000012, + DT_GNU_HASH = 0x6FFFFEF5, // Reference to the GNU hash table. DT_TLSDESC_PLT = 0x6FFFFEF6, // Location of PLT entry for TLS descriptor resolver calls. @@ -1324,6 +1382,14 @@ enum { NT_GNU_GOLD_VERSION = 4, }; +// AMDGPU specific notes. +enum { + // Note types with values between 0 and 9 (inclusive) are reserved. + NT_AMD_AMDGPU_HSA_METADATA = 10, + NT_AMD_AMDGPU_ISA = 11, + NT_AMD_AMDGPU_PAL_METADATA = 12 +}; + enum { GNU_ABI_TAG_LINUX = 0, GNU_ABI_TAG_HURD = 1, @@ -1334,6 +1400,14 @@ enum { GNU_ABI_TAG_NACL = 6, }; +// Android packed relocation group flags. +enum { + RELOCATION_GROUPED_BY_INFO_FLAG = 1, + RELOCATION_GROUPED_BY_OFFSET_DELTA_FLAG = 2, + RELOCATION_GROUPED_BY_ADDEND_FLAG = 4, + RELOCATION_GROUP_HAS_ADDEND_FLAG = 8, +}; + // Compressed section header for ELF32. struct Elf32_Chdr { Elf32_Word ch_type; diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AMDGPU.def b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AMDGPU.def index c66f88d14ec7..00b19c4161d0 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AMDGPU.def +++ b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AMDGPU.def @@ -14,3 +14,4 @@ ELF_RELOC(R_AMDGPU_GOTPCREL32_LO, 8) ELF_RELOC(R_AMDGPU_GOTPCREL32_HI, 9) ELF_RELOC(R_AMDGPU_REL32_LO, 10) ELF_RELOC(R_AMDGPU_REL32_HI, 11) +ELF_RELOC(R_AMDGPU_RELATIVE64, 13) diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/ARC.def b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/ARC.def new file mode 100644 index 000000000000..5691fb3458e2 --- /dev/null +++ b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/ARC.def @@ -0,0 +1,74 @@ + +#ifndef ELF_RELOC +#error "ELF_RELOC must be defined" +#endif + +ELF_RELOC(R_ARC_NONE, 0) +ELF_RELOC(R_ARC_8, 1) +ELF_RELOC(R_ARC_16, 2) +ELF_RELOC(R_ARC_24, 3) +ELF_RELOC(R_ARC_32, 4) +ELF_RELOC(R_ARC_N8, 8) +ELF_RELOC(R_ARC_N16, 9) +ELF_RELOC(R_ARC_N24, 10) +ELF_RELOC(R_ARC_N32, 11) +ELF_RELOC(R_ARC_SDA, 12) +ELF_RELOC(R_ARC_SECTOFF, 13) +ELF_RELOC(R_ARC_S21H_PCREL, 14) +ELF_RELOC(R_ARC_S21W_PCREL, 15) +ELF_RELOC(R_ARC_S25H_PCREL, 16) +ELF_RELOC(R_ARC_S25W_PCREL, 17) +ELF_RELOC(R_ARC_SDA32, 18) +ELF_RELOC(R_ARC_SDA_LDST, 19) +ELF_RELOC(R_ARC_SDA_LDST1, 20) +ELF_RELOC(R_ARC_SDA_LDST2, 21) +ELF_RELOC(R_ARC_SDA16_LD, 22) +ELF_RELOC(R_ARC_SDA16_LD1, 23) +ELF_RELOC(R_ARC_SDA16_LD2, 24) +ELF_RELOC(R_ARC_S13_PCREL, 25) +ELF_RELOC(R_ARC_W, 26) +ELF_RELOC(R_ARC_32_ME, 27) +ELF_RELOC(R_ARC_32_ME_S, 105) +ELF_RELOC(R_ARC_N32_ME, 28) +ELF_RELOC(R_ARC_SECTOFF_ME, 29) +ELF_RELOC(R_ARC_SDA32_ME, 30) +ELF_RELOC(R_ARC_W_ME, 31) +ELF_RELOC(R_AC_SECTOFF_U8, 35) +ELF_RELOC(R_AC_SECTOFF_U8_1, 36) +ELF_RELOC(R_AC_SECTOFF_U8_2, 37) +ELF_RELOC(R_AC_SECTOFF_S9, 38) +ELF_RELOC(R_AC_SECTOFF_S9_1, 39) +ELF_RELOC(R_AC_SECTOFF_S9_2, 40) +ELF_RELOC(R_ARC_SECTOFF_ME_1, 41) +ELF_RELOC(R_ARC_SECTOFF_ME_2, 42) +ELF_RELOC(R_ARC_SECTOFF_1, 43) +ELF_RELOC(R_ARC_SECTOFF_2, 44) +ELF_RELOC(R_ARC_SDA_12, 45) +ELF_RELOC(R_ARC_SDA16_ST2, 48) +ELF_RELOC(R_ARC_32_PCREL, 49) +ELF_RELOC(R_ARC_PC32, 50) +ELF_RELOC(R_ARC_GOT32, 59) +ELF_RELOC(R_ARC_GOTPC32, 51) +ELF_RELOC(R_ARC_PLT32, 52) +ELF_RELOC(R_ARC_COPY, 53) +ELF_RELOC(R_ARC_GLOB_DAT, 54) +ELF_RELOC(R_ARC_JMP_SLOT, 55) +ELF_RELOC(R_ARC_RELATIVE, 56) +ELF_RELOC(R_ARC_GOTOFF, 57) +ELF_RELOC(R_ARC_GOTPC, 58) +ELF_RELOC(R_ARC_S21W_PCREL_PLT, 60) +ELF_RELOC(R_ARC_S25H_PCREL_PLT, 61) +ELF_RELOC(R_ARC_JLI_SECTOFF, 63) +ELF_RELOC(R_ARC_TLS_DTPMOD, 66) +ELF_RELOC(R_ARC_TLS_TPOFF, 68) +ELF_RELOC(R_ARC_TLS_GD_GOT, 69) +ELF_RELOC(R_ARC_TLS_GD_LD, 70) +ELF_RELOC(R_ARC_TLS_GD_CALL, 71) +ELF_RELOC(R_ARC_TLS_IE_GOT, 72) +ELF_RELOC(R_ARC_TLS_DTPOFF, 67) +ELF_RELOC(R_ARC_TLS_DTPOFF_S9, 73) +ELF_RELOC(R_ARC_TLS_LE_S9, 74) +ELF_RELOC(R_ARC_TLS_LE_32, 75) +ELF_RELOC(R_ARC_S25W_PCREL_PLT, 76) +ELF_RELOC(R_ARC_S21H_PCREL_PLT, 77) +ELF_RELOC(R_ARC_NPS_CMEM16, 78) diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AVR.def b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AVR.def index 5692d6cb9aa0..696fc60b0f5a 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AVR.def +++ b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/AVR.def @@ -33,8 +33,9 @@ ELF_RELOC(R_AVR_8, 26) ELF_RELOC(R_AVR_8_LO8, 27) ELF_RELOC(R_AVR_8_HI8, 28) ELF_RELOC(R_AVR_8_HLO8, 29) -ELF_RELOC(R_AVR_SYM_DIFF, 30) -ELF_RELOC(R_AVR_16_LDST, 31) +ELF_RELOC(R_AVR_DIFF8, 30) +ELF_RELOC(R_AVR_DIFF16, 31) +ELF_RELOC(R_AVR_DIFF32, 32) ELF_RELOC(R_AVR_LDS_STS_16, 33) ELF_RELOC(R_AVR_PORT6, 34) ELF_RELOC(R_AVR_PORT5, 35) diff --git a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def index 9ec4955d26db..5cc4c0ec3029 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def +++ b/contrib/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV.def @@ -48,3 +48,12 @@ ELF_RELOC(R_RISCV_RVC_JUMP, 45) ELF_RELOC(R_RISCV_RVC_LUI, 46) ELF_RELOC(R_RISCV_GPREL_I, 47) ELF_RELOC(R_RISCV_GPREL_S, 48) +ELF_RELOC(R_RISCV_TPREL_I, 49) +ELF_RELOC(R_RISCV_TPREL_S, 50) +ELF_RELOC(R_RISCV_RELAX, 51) +ELF_RELOC(R_RISCV_SUB6, 52) +ELF_RELOC(R_RISCV_SET6, 53) +ELF_RELOC(R_RISCV_SET8, 54) +ELF_RELOC(R_RISCV_SET16, 55) +ELF_RELOC(R_RISCV_SET32, 56) +ELF_RELOC(R_RISCV_32_PCREL, 57) diff --git a/contrib/llvm/include/llvm/BinaryFormat/MachO.h b/contrib/llvm/include/llvm/BinaryFormat/MachO.h index 3529c72acd9d..060fbe162ad2 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/MachO.h +++ b/contrib/llvm/include/llvm/BinaryFormat/MachO.h @@ -481,7 +481,7 @@ enum RelocationInfoType { enum { VM_PROT_READ = 0x1, VM_PROT_WRITE = 0x2, VM_PROT_EXECUTE = 0x4 }; // Values for platform field in build_version_command. -enum { +enum PlatformType { PLATFORM_MACOS = 1, PLATFORM_IOS = 2, PLATFORM_TVOS = 3, @@ -1373,19 +1373,19 @@ inline void swapStruct(fvmlib_command &C) { // Get/Set functions from <mach-o/nlist.h> -static inline uint16_t GET_LIBRARY_ORDINAL(uint16_t n_desc) { +inline uint16_t GET_LIBRARY_ORDINAL(uint16_t n_desc) { return (((n_desc) >> 8u) & 0xffu); } -static inline void SET_LIBRARY_ORDINAL(uint16_t &n_desc, uint8_t ordinal) { +inline void SET_LIBRARY_ORDINAL(uint16_t &n_desc, uint8_t ordinal) { n_desc = (((n_desc)&0x00ff) | (((ordinal)&0xff) << 8)); } -static inline uint8_t GET_COMM_ALIGN(uint16_t n_desc) { +inline uint8_t GET_COMM_ALIGN(uint16_t n_desc) { return (n_desc >> 8u) & 0x0fu; } -static inline void SET_COMM_ALIGN(uint16_t &n_desc, uint8_t align) { +inline void SET_COMM_ALIGN(uint16_t &n_desc, uint8_t align) { n_desc = ((n_desc & 0xf0ffu) | ((align & 0x0fu) << 8u)); } @@ -1449,15 +1449,13 @@ enum CPUSubTypeX86 { CPU_SUBTYPE_X86_ARCH1 = 4, CPU_SUBTYPE_X86_64_H = 8 }; -static inline int CPU_SUBTYPE_INTEL(int Family, int Model) { +inline int CPU_SUBTYPE_INTEL(int Family, int Model) { return Family | (Model << 4); } -static inline int CPU_SUBTYPE_INTEL_FAMILY(CPUSubTypeX86 ST) { +inline int CPU_SUBTYPE_INTEL_FAMILY(CPUSubTypeX86 ST) { return ((int)ST) & 0x0f; } -static inline int CPU_SUBTYPE_INTEL_MODEL(CPUSubTypeX86 ST) { - return ((int)ST) >> 4; -} +inline int CPU_SUBTYPE_INTEL_MODEL(CPUSubTypeX86 ST) { return ((int)ST) >> 4; } enum { CPU_SUBTYPE_INTEL_FAMILY_MAX = 15, CPU_SUBTYPE_INTEL_MODEL_ALL = 0 }; enum CPUSubTypeARM { diff --git a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h index 23e30b7a868d..506cd0393e9a 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/contrib/llvm/include/llvm/BinaryFormat/Wasm.h @@ -91,12 +91,17 @@ struct WasmLocalDecl { struct WasmFunction { std::vector<WasmLocalDecl> Locals; ArrayRef<uint8_t> Body; + uint32_t CodeSectionOffset; + uint32_t Size; }; struct WasmDataSegment { uint32_t MemoryIndex; WasmInitExpr Offset; ArrayRef<uint8_t> Content; + StringRef Name; + uint32_t Alignment; + uint32_t Flags; }; struct WasmElemSegment { @@ -112,9 +117,14 @@ struct WasmRelocation { int64_t Addend; // A value to add to the symbol. }; +struct WasmInitFunc { + uint32_t Priority; + uint32_t FunctionIndex; +}; + struct WasmLinkingData { uint32_t DataSize; - uint32_t DataAlignment; + std::vector<WasmInitFunc> InitFunctions; }; enum : unsigned { @@ -180,15 +190,20 @@ enum class ValType { // Linking metadata kinds. enum : unsigned { - WASM_STACK_POINTER = 0x1, WASM_SYMBOL_INFO = 0x2, WASM_DATA_SIZE = 0x3, - WASM_DATA_ALIGNMENT = 0x4, + WASM_SEGMENT_INFO = 0x5, + WASM_INIT_FUNCS = 0x6, }; -enum : unsigned { - WASM_SYMBOL_FLAG_WEAK = 0x1, -}; +const unsigned WASM_SYMBOL_BINDING_MASK = 0x3; +const unsigned WASM_SYMBOL_VISIBILITY_MASK = 0x4; + +const unsigned WASM_SYMBOL_BINDING_GLOBAL = 0x0; +const unsigned WASM_SYMBOL_BINDING_WEAK = 0x1; +const unsigned WASM_SYMBOL_BINDING_LOCAL = 0x2; +const unsigned WASM_SYMBOL_VISIBILITY_DEFAULT = 0x0; +const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4; #define WASM_RELOC(name, value) name = value, diff --git a/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs/WebAssembly.def b/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs/WebAssembly.def index da64e025478d..d6f0e42b33bf 100644 --- a/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs/WebAssembly.def +++ b/contrib/llvm/include/llvm/BinaryFormat/WasmRelocs/WebAssembly.def @@ -6,8 +6,8 @@ WASM_RELOC(R_WEBASSEMBLY_FUNCTION_INDEX_LEB, 0) WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_SLEB, 1) WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_I32, 2) -WASM_RELOC(R_WEBASSEMBLY_GLOBAL_ADDR_LEB, 3) -WASM_RELOC(R_WEBASSEMBLY_GLOBAL_ADDR_SLEB, 4) -WASM_RELOC(R_WEBASSEMBLY_GLOBAL_ADDR_I32, 5) +WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_LEB, 3) +WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_SLEB, 4) +WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_I32, 5) WASM_RELOC(R_WEBASSEMBLY_TYPE_INDEX_LEB, 6) WASM_RELOC(R_WEBASSEMBLY_GLOBAL_INDEX_LEB, 7) diff --git a/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h b/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h index 160ddad5761f..ce8bdd9cf0b4 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/contrib/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -1,4 +1,4 @@ -//===-- llvm/Bitcode/BitcodeReader.h - Bitcode reader ----*- C++ -*-===// +//===- llvm/Bitcode/BitcodeReader.h - Bitcode reader ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,18 +14,23 @@ #ifndef LLVM_BITCODE_BITCODEREADER_H #define LLVM_BITCODE_BITCODEREADER_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Bitcode/BitCodes.h" -#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" +#include <cstdint> #include <memory> - +#include <string> +#include <system_error> +#include <vector> namespace llvm { - class LLVMContext; - class Module; + +class LLVMContext; +class Module; // These functions are for converting Expected/Error values to // ErrorOr/std::error_code for compatibility with legacy clients. FIXME: @@ -81,6 +86,7 @@ namespace llvm { StringRef getBuffer() const { return StringRef((const char *)Buffer.begin(), Buffer.size()); } + StringRef getStrtab() const { return Strtab; } StringRef getModuleIdentifier() const { return ModuleIdentifier; } @@ -182,7 +188,6 @@ namespace llvm { /// isBitcodeWrapper - Return true if the given bytes are the magic bytes /// for an LLVM IR bitcode wrapper. - /// inline bool isBitcodeWrapper(const unsigned char *BufPtr, const unsigned char *BufEnd) { // See if you can find the hidden message in the magic bytes :-). @@ -196,7 +201,6 @@ namespace llvm { /// isRawBitcode - Return true if the given bytes are the magic bytes for /// raw LLVM IR bitcode (without a wrapper). - /// inline bool isRawBitcode(const unsigned char *BufPtr, const unsigned char *BufEnd) { // These bytes sort of have a hidden message, but it's not in @@ -210,7 +214,6 @@ namespace llvm { /// isBitcode - Return true if the given bytes are the magic bytes for /// LLVM IR bitcode, either with or without a wrapper. - /// inline bool isBitcode(const unsigned char *BufPtr, const unsigned char *BufEnd) { return isBitcodeWrapper(BufPtr, BufEnd) || @@ -258,10 +261,12 @@ namespace llvm { return std::error_code(static_cast<int>(E), BitcodeErrorCategory()); } -} // End llvm namespace +} // end namespace llvm namespace std { + template <> struct is_error_code_enum<llvm::BitcodeError> : std::true_type {}; -} -#endif +} // end namespace std + +#endif // LLVM_BITCODE_BITCODEREADER_H diff --git a/contrib/llvm/include/llvm/Bitcode/BitcodeWriter.h b/contrib/llvm/include/llvm/Bitcode/BitcodeWriter.h index f8b7fb341e88..c78077525c8b 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitcodeWriter.h +++ b/contrib/llvm/include/llvm/Bitcode/BitcodeWriter.h @@ -1,4 +1,4 @@ -//===-- llvm/Bitcode/BitcodeWriter.h - Bitcode writers ----*- C++ -*-===// +//===- llvm/Bitcode/BitcodeWriter.h - Bitcode writers -----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,14 +14,20 @@ #ifndef LLVM_BITCODE_BITCODEWRITER_H #define LLVM_BITCODE_BITCODEWRITER_H +#include "llvm/ADT/StringRef.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/MC/StringTableBuilder.h" +#include "llvm/Support/Allocator.h" +#include <map> +#include <memory> #include <string> +#include <vector> namespace llvm { - class BitstreamWriter; - class Module; - class raw_ostream; + +class BitstreamWriter; +class Module; +class raw_ostream; class BitcodeWriter { SmallVectorImpl<char> &Buffer; @@ -39,7 +45,7 @@ namespace llvm { std::vector<Module *> Mods; - public: + public: /// Create a BitcodeWriter that writes to Buffer. BitcodeWriter(SmallVectorImpl<char> &Buffer); @@ -84,6 +90,16 @@ namespace llvm { const ModuleSummaryIndex *Index = nullptr, bool GenerateHash = false, ModuleHash *ModHash = nullptr); + /// Write the specified thin link bitcode file (i.e., the minimized bitcode + /// file) to the buffer specified at construction time. The thin link + /// bitcode file is used for thin link, and it only contains the necessary + /// information for thin link. + /// + /// ModHash is for use in ThinLTO incremental build, generated while the + /// IR bitcode file writing. + void writeThinLinkBitcode(const Module *M, const ModuleSummaryIndex &Index, + const ModuleHash &ModHash); + void writeIndex( const ModuleSummaryIndex *Index, const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex); @@ -116,6 +132,17 @@ namespace llvm { bool GenerateHash = false, ModuleHash *ModHash = nullptr); + /// Write the specified thin link bitcode file (i.e., the minimized bitcode + /// file) to the given raw output stream, where it will be written in a new + /// bitcode block. The thin link bitcode file is used for thin link, and it + /// only contains the necessary information for thin link. + /// + /// ModHash is for use in ThinLTO incremental build, generated while the IR + /// bitcode file writing. + void WriteThinLinkBitcodeToFile(const Module *M, raw_ostream &Out, + const ModuleSummaryIndex &Index, + const ModuleHash &ModHash); + /// Write the specified module summary index to the given raw output stream, /// where it will be written in a new bitcode block. This is used when /// writing the combined index file for ThinLTO. When writing a subset of the @@ -124,6 +151,7 @@ namespace llvm { void WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex = nullptr); -} // End llvm namespace -#endif +} // end namespace llvm + +#endif // LLVM_BITCODE_BITCODEWRITER_H diff --git a/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h b/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h index fc06eeefbf26..b484fa2efbfb 100644 --- a/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h +++ b/contrib/llvm/include/llvm/Bitcode/BitstreamReader.h @@ -43,7 +43,7 @@ public: unsigned BlockID; std::vector<std::shared_ptr<BitCodeAbbrev>> Abbrevs; std::string Name; - std::vector<std::pair<unsigned, std::string> > RecordNames; + std::vector<std::pair<unsigned, std::string>> RecordNames; }; private: @@ -88,7 +88,7 @@ public: /// follow the word size of the host machine for efficiency. We use word_t in /// places that are aware of this to make it perfectly explicit what is going /// on. - typedef size_t word_t; + using word_t = size_t; private: word_t CurWord = 0; diff --git a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 3777f956cf27..70194c043479 100644 --- a/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/contrib/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -558,7 +558,9 @@ enum AttributeKindCodes { ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY = 50, ATTR_KIND_ALLOC_SIZE = 51, ATTR_KIND_WRITEONLY = 52, - ATTR_KIND_SPECULATABLE = 53 + ATTR_KIND_SPECULATABLE = 53, + ATTR_KIND_STRICT_FP = 54, + ATTR_KIND_SANITIZE_HWADDRESS = 55, }; enum ComdatSelectionKindCodes { diff --git a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h index 60bbc9aaa5bd..b8944a668000 100644 --- a/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/contrib/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -43,11 +43,11 @@ class DIE; class DIEAbbrev; class DwarfDebug; class GCMetadataPrinter; +class GCStrategy; class GlobalIndirectSymbol; class GlobalObject; class GlobalValue; class GlobalVariable; -class GCStrategy; class MachineBasicBlock; class MachineConstantPoolValue; class MachineFunction; @@ -58,6 +58,7 @@ class MachineModuleInfo; class MachineOptimizationRemarkEmitter; class MCAsmInfo; class MCCFIInstruction; +struct MCCodePaddingContext; class MCContext; class MCExpr; class MCInst; @@ -76,11 +77,9 @@ class TargetMachine; class AsmPrinter : public MachineFunctionPass { public: /// Target machine description. - /// TargetMachine &TM; /// Target Asm Printer information. - /// const MCAsmInfo *MAI; /// This is the context for the output file that we are streaming. This owns @@ -103,7 +102,6 @@ public: /// The symbol for the current function. This is recalculated at the beginning /// of each call to runOnMachineFunction(). - /// MCSymbol *CurrentFnSym = nullptr; /// The symbol used to represent the start of the current function for the @@ -116,7 +114,7 @@ public: using GOTEquivUsePair = std::pair<const GlobalVariable *, unsigned>; MapVector<const MCSymbol *, GOTEquivUsePair> GlobalGOTEquivs; - /// Enable print [latency:throughput] in output + /// Enable print [latency:throughput] in output. bool EnablePrintSchedInfo = false; private: @@ -128,8 +126,8 @@ private: void *GCMetadataPrinters = nullptr; // Really a DenseMap. /// Emit comments in assembly output if this is true. - /// bool VerboseAsm; + static char ID; /// If VerboseAsm is set, a pointer to the loop info for this function. @@ -149,6 +147,7 @@ private: TimerDescription(TimerDescription), TimerGroupName(TimerGroupName), TimerGroupDescription(TimerGroupDescription) {} }; + /// A vector of all debug/EH info emitters we should use. This vector /// maintains ownership of the emitters. SmallVector<HandlerInfo, 1> Handlers; @@ -187,11 +186,9 @@ public: bool isPositionIndependent() const; /// Return true if assembly output should contain comments. - /// bool isVerbose() const { return VerboseAsm; } /// Return a unique ID for the current function. - /// unsigned getFunctionNumber() const; MCSymbol *getFunctionBegin() const { return CurrentFnBegin; } @@ -235,13 +232,15 @@ public: // The table will contain these structs that point to the sled, the function // containing the sled, and what kind of sled (and whether they should always - // be instrumented). + // be instrumented). We also use a version identifier that the runtime can use + // to decide what to do with the sled, depending on the version of the sled. struct XRayFunctionEntry { const MCSymbol *Sled; const MCSymbol *Function; SledKind Kind; bool AlwaysInstrument; const class Function *Fn; + uint8_t Version; void emit(int, MCStreamer *, const MCSymbol *) const; }; @@ -249,8 +248,12 @@ public: // All the sleds to be emitted. SmallVector<XRayFunctionEntry, 4> Sleds; + // A unique ID used for ELF sections associated with a particular function. + unsigned XRayFnUniqueID = 0; + // Helper function to record a given XRay sled. - void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind); + void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind, + uint8_t Version = 0); /// Emit a table with all XRay instrumentation points. void emitXRayTable(); @@ -260,7 +263,6 @@ public: //===------------------------------------------------------------------===// /// Record analysis usage. - /// void getAnalysisUsage(AnalysisUsage &AU) const override; /// Set up the AsmPrinter when we are working on a new module. If your pass @@ -293,8 +295,10 @@ public: void emitFrameAlloc(const MachineInstr &MI); + void emitStackSizeSection(const MachineFunction &MF); + enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; - CFIMoveType needsCFIMoves(); + CFIMoveType needsCFIMoves() const; /// Returns false if needsCFIMoves() == CFI_M_EH for any function /// in the module. @@ -305,12 +309,10 @@ public: /// Print to the current output stream assembly representations of the /// constants in the constant pool MCP. This is used to print out constants /// which have been "spilled to memory" by the code generator. - /// virtual void EmitConstantPool(); /// Print assembly representations of the jump tables used by the current /// function to the current output stream. - /// virtual void EmitJumpTableInfo(); /// Emit the specified global variable to the .s file. @@ -325,7 +327,6 @@ public: /// global value is specified, and if that global has an explicit alignment /// requested, it will override the alignment request if required for /// correctness. - /// void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const; /// Lower the specified LLVM Constant to an MCExpr. @@ -379,7 +380,7 @@ public: virtual void EmitBasicBlockStart(const MachineBasicBlock &MBB) const; /// Targets can override this to emit stuff at the end of a basic block. - virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB) {} + virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB); /// Targets should implement this to emit instructions. virtual void EmitInstruction(const MachineInstr *) { @@ -443,15 +444,12 @@ public: void printOffset(int64_t Offset, raw_ostream &OS) const; /// Emit a byte directive and value. - /// void EmitInt8(int Value) const; /// Emit a short directive and value. - /// void EmitInt16(int Value) const; /// Emit a long directive and value. - /// void EmitInt32(int Value) const; /// Emit something like ".long Hi-Lo" where the size in bytes of the directive @@ -481,8 +479,12 @@ public: void EmitSLEB128(int64_t Value, const char *Desc = nullptr) const; /// Emit the specified unsigned leb128 value. - void EmitULEB128(uint64_t Value, const char *Desc = nullptr, - unsigned PadTo = 0) const; + void EmitULEB128(uint64_t Value, const char *Desc = nullptr) const; + + /// Emit the specified unsigned leb128 value padded to a specific number + /// bytes + void EmitPaddedULEB128(uint64_t Value, unsigned PadTo, + const char *Desc = nullptr) const; /// Emit a .byte 42 directive that corresponds to an encoding. If verbose /// assembly output is enabled, we output comments describing the encoding. @@ -622,10 +624,13 @@ private: void EmitModuleIdents(Module &M); void EmitXXStructorList(const DataLayout &DL, const Constant *List, bool isCtor); + GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C); /// Emit GlobalAlias or GlobalIFunc. void emitGlobalIndirectSymbol(Module &M, const GlobalIndirectSymbol& GIS); + void setupCodePaddingContext(const MachineBasicBlock &MBB, + MCCodePaddingContext &Context) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 633107024792..bb5e7f9e8e30 100644 --- a/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -6,25 +6,63 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides a helper that implements much of the TTI interface in /// terms of the target-independent code generator and TargetLowering /// interfaces. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_BASICTTIIMPL_H #define LLVM_CODEGEN_BASICTTIIMPL_H +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TargetTransformInfoImpl.h" +#include "llvm/CodeGen/ISDOpcodes.h" +#include "llvm/CodeGen/MachineValueType.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/MC/MCSchedule.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <limits> +#include <utility> namespace llvm { +class Function; +class GlobalValue; +class LLVMContext; +class ScalarEvolution; +class SCEV; +class TargetMachine; + extern cl::opt<unsigned> PartialUnrollingThreshold; /// \brief Base class which can be used to help build a TTI implementation. @@ -39,8 +77,8 @@ extern cl::opt<unsigned> PartialUnrollingThreshold; template <typename T> class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> { private: - typedef TargetTransformInfoImplCRTPBase<T> BaseT; - typedef TargetTransformInfo TTI; + using BaseT = TargetTransformInfoImplCRTPBase<T>; + using TTI = TargetTransformInfo; /// Estimate a cost of shuffle as a sequence of extract and insert /// operations. @@ -110,13 +148,13 @@ public: bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, - unsigned AddrSpace) { + unsigned AddrSpace, Instruction *I = nullptr) { TargetLoweringBase::AddrMode AM; AM.BaseGV = BaseGV; AM.BaseOffs = BaseOffset; AM.HasBaseReg = HasBaseReg; AM.Scale = Scale; - return getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace); + return getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace, I); } bool isLSRCostLess(TTI::LSRCost C1, TTI::LSRCost C2) { @@ -133,10 +171,6 @@ public: return getTLI()->getScalingFactorCost(DL, AM, Ty, AddrSpace); } - bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) { - return getTLI()->isFoldableMemAccessOffset(I, Offset); - } - bool isTruncateFree(Type *Ty1, Type *Ty2) { return getTLI()->isTruncateFree(Ty1, Ty2); } @@ -235,7 +269,8 @@ public: if (N < 2 || N < TLI->getMinimumJumpTableEntries()) return N; uint64_t Range = - (MaxCaseVal - MinCaseVal).getLimitedValue(UINT64_MAX - 1) + 1; + (MaxCaseVal - MinCaseVal) + .getLimitedValue(std::numeric_limits<uint64_t>::max() - 1) + 1; // Check whether a range of clusters is dense enough for a jump table if (TLI->isSuitableForJumpTable(&SI, N, Range)) { JumpTableSize = Range; @@ -262,6 +297,10 @@ public: TLI->isOperationLegalOrCustom(ISD::FSQRT, VT); } + bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) { + return true; + } + unsigned getFPOpCost(Type *Ty) { // By default, FP instructions are no more expensive since they are // implemented in HW. Target specific TTI can override this. @@ -272,17 +311,15 @@ public: const TargetLoweringBase *TLI = getTLI(); switch (Opcode) { default: break; - case Instruction::Trunc: { + case Instruction::Trunc: if (TLI->isTruncateFree(OpTy, Ty)) return TargetTransformInfo::TCC_Free; return TargetTransformInfo::TCC_Basic; - } - case Instruction::ZExt: { + case Instruction::ZExt: if (TLI->isZExtFree(OpTy, Ty)) return TargetTransformInfo::TCC_Free; return TargetTransformInfo::TCC_Basic; } - } return BaseT::getOperationCost(Opcode, Ty, OpTy); } @@ -354,6 +391,13 @@ public: UP.BEInsns = 2; } + int getInstructionLatency(const Instruction *I) { + if (isa<LoadInst>(I)) + return getST()->getSchedModel().DefaultLoadLatency; + + return BaseT::getInstructionLatency(I); + } + /// @} /// \name Vector TTI Implementations @@ -394,8 +438,8 @@ public: if (A->getType()->isVectorTy()) { VecTy = A->getType(); // If A is a vector operand, VF should be 1 or correspond to A. - assert ((VF == 1 || VF == VecTy->getVectorNumElements()) && - "Vector argument does not match VF"); + assert((VF == 1 || VF == VecTy->getVectorNumElements()) && + "Vector argument does not match VF"); } else VecTy = VectorType::get(A->getType(), VF); @@ -408,8 +452,8 @@ public: } unsigned getScalarizationOverhead(Type *VecTy, ArrayRef<const Value *> Args) { - assert (VecTy->isVectorTy()); - + assert(VecTy->isVectorTy()); + unsigned Cost = 0; Cost += getScalarizationOverhead(VecTy, true, false); @@ -531,7 +575,6 @@ public: // Handle scalar conversions. if (!Src->isVectorTy() && !Dst->isVectorTy()) { - // Scalar bitcasts are usually free. if (Opcode == Instruction::BitCast) return 0; @@ -547,7 +590,6 @@ public: // Check vector-to-vector casts. if (Dst->isVectorTy() && Src->isVectorTy()) { - // If the cast is between same-sized registers, then the check is simple. if (SrcLT.first == DstLT.first && SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) { @@ -743,7 +785,6 @@ public: // We only scale the cost of loads since interleaved store groups aren't // allowed to have gaps. if (Opcode == Instruction::Load && VecTySize > VecTyLTSize) { - // The number of loads of a legal type it will take to represent a load // of the unlegalized vector type. unsigned NumLegalInsts = ceil(VecTySize, VecTyLTSize); @@ -821,7 +862,7 @@ public: ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF = 1) { unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1); - assert ((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type"); + assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type"); switch (IID) { default: { @@ -829,7 +870,7 @@ public: SmallVector<Type *, 4> Types; for (Value *Op : Args) { Type *OpTy = Op->getType(); - assert (VF == 1 || !OpTy->isVectorTy()); + assert(VF == 1 || !OpTy->isVectorTy()); Types.push_back(VF == 1 ? OpTy : VectorType::get(OpTy, VF)); } @@ -839,7 +880,7 @@ public: // Compute the scalarization overhead based on Args for a vector // intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while // CostModel will pass a vector RetTy and VF is 1. - unsigned ScalarizationCost = UINT_MAX; + unsigned ScalarizationCost = std::numeric_limits<unsigned>::max(); if (RetVF > 1 || VF > 1) { ScalarizationCost = 0; if (!RetTy->isVoidTy()) @@ -851,7 +892,7 @@ public: getIntrinsicInstrCost(IID, RetTy, Types, FMF, ScalarizationCost); } case Intrinsic::masked_scatter: { - assert (VF == 1 && "Can't vectorize types here."); + assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[3]; bool VarMask = !isa<Constant>(Mask); unsigned Alignment = cast<ConstantInt>(Args[2])->getZExtValue(); @@ -862,7 +903,7 @@ public: Alignment); } case Intrinsic::masked_gather: { - assert (VF == 1 && "Can't vectorize types here."); + assert(VF == 1 && "Can't vectorize types here."); Value *Mask = Args[2]; bool VarMask = !isa<Constant>(Mask); unsigned Alignment = cast<ConstantInt>(Args[1])->getZExtValue(); @@ -873,13 +914,14 @@ public: } } } - + /// Get intrinsic cost based on argument types. - /// If ScalarizationCostPassed is UINT_MAX, the cost of scalarizing the - /// arguments and the return value will be computed based on types. - unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<Type *> Tys, FastMathFlags FMF, - unsigned ScalarizationCostPassed = UINT_MAX) { + /// If ScalarizationCostPassed is std::numeric_limits<unsigned>::max(), the + /// cost of scalarizing the arguments and the return value will be computed + /// based on types. + unsigned getIntrinsicInstrCost( + Intrinsic::ID IID, Type *RetTy, ArrayRef<Type *> Tys, FastMathFlags FMF, + unsigned ScalarizationCostPassed = std::numeric_limits<unsigned>::max()) { SmallVector<unsigned, 2> ISDs; unsigned SingleCallCost = 10; // Library call cost. Make it expensive. switch (IID) { @@ -889,7 +931,7 @@ public: unsigned ScalarCalls = 1; Type *ScalarRetTy = RetTy; if (RetTy->isVectorTy()) { - if (ScalarizationCostPassed == UINT_MAX) + if (ScalarizationCostPassed == std::numeric_limits<unsigned>::max()) ScalarizationCost = getScalarizationOverhead(RetTy, true, false); ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); ScalarRetTy = RetTy->getScalarType(); @@ -898,7 +940,7 @@ public: for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { Type *Ty = Tys[i]; if (Ty->isVectorTy()) { - if (ScalarizationCostPassed == UINT_MAX) + if (ScalarizationCostPassed == std::numeric_limits<unsigned>::max()) ScalarizationCost += getScalarizationOverhead(Ty, false, true); ScalarCalls = std::max(ScalarCalls, Ty->getVectorNumElements()); Ty = Ty->getScalarType(); @@ -985,6 +1027,7 @@ public: // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free. case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: + case Intrinsic::sideeffect: return 0; case Intrinsic::masked_store: return static_cast<T *>(this) @@ -1047,8 +1090,10 @@ public: // this will emit a costly libcall, adding call overhead and spills. Make it // very expensive. if (RetTy->isVectorTy()) { - unsigned ScalarizationCost = ((ScalarizationCostPassed != UINT_MAX) ? - ScalarizationCostPassed : getScalarizationOverhead(RetTy, true, false)); + unsigned ScalarizationCost = + ((ScalarizationCostPassed != std::numeric_limits<unsigned>::max()) + ? ScalarizationCostPassed + : getScalarizationOverhead(RetTy, true, false)); unsigned ScalarCalls = RetTy->getVectorNumElements(); SmallVector<Type *, 4> ScalarTys; for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { @@ -1061,7 +1106,7 @@ public: IID, RetTy->getScalarType(), ScalarTys, FMF); for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { if (Tys[i]->isVectorTy()) { - if (ScalarizationCostPassed == UINT_MAX) + if (ScalarizationCostPassed == std::numeric_limits<unsigned>::max()) ScalarizationCost += getScalarizationOverhead(Tys[i], false, true); ScalarCalls = std::max(ScalarCalls, Tys[i]->getVectorNumElements()); } @@ -1096,7 +1141,7 @@ public: unsigned getAddressComputationCost(Type *Ty, ScalarEvolution *, const SCEV *) { - return 0; + return 0; } /// Try to calculate arithmetic and shuffle op costs for reduction operations. @@ -1134,7 +1179,8 @@ public: /// /// The cost model should take into account that the actual length of the /// vector is reduced on each iteration. - unsigned getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise) { + unsigned getArithmeticReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwise) { assert(Ty->isVectorTy() && "Expect a vector type"); Type *ScalarTy = Ty->getVectorElementType(); unsigned NumVecElts = Ty->getVectorNumElements(); @@ -1159,7 +1205,7 @@ public: } // The minimal length of the vector is limited by the real length of vector // operations performed on the current platform. That's why several final - // reduction opertions are perfomed on the vectors with the same + // reduction operations are performed on the vectors with the same // architecture-dependent length. ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, @@ -1169,6 +1215,66 @@ public: return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true); } + /// Try to calculate op costs for min/max reduction operations. + /// \param CondTy Conditional type for the Select instruction. + unsigned getMinMaxReductionCost(Type *Ty, Type *CondTy, bool IsPairwise, + bool) { + assert(Ty->isVectorTy() && "Expect a vector type"); + Type *ScalarTy = Ty->getVectorElementType(); + Type *ScalarCondTy = CondTy->getVectorElementType(); + unsigned NumVecElts = Ty->getVectorNumElements(); + unsigned NumReduxLevels = Log2_32(NumVecElts); + unsigned CmpOpcode; + if (Ty->isFPOrFPVectorTy()) { + CmpOpcode = Instruction::FCmp; + } else { + assert(Ty->isIntOrIntVectorTy() && + "expecting floating point or integer type for min/max reduction"); + CmpOpcode = Instruction::ICmp; + } + unsigned MinMaxCost = 0; + unsigned ShuffleCost = 0; + auto *ConcreteTTI = static_cast<T *>(this); + std::pair<unsigned, MVT> LT = + ConcreteTTI->getTLI()->getTypeLegalizationCost(DL, Ty); + unsigned LongVectorCount = 0; + unsigned MVTLen = + LT.second.isVector() ? LT.second.getVectorNumElements() : 1; + while (NumVecElts > MVTLen) { + NumVecElts /= 2; + // Assume the pairwise shuffles add a cost. + ShuffleCost += (IsPairwise + 1) * + ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, + NumVecElts, Ty); + MinMaxCost += + ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, + nullptr); + Ty = VectorType::get(ScalarTy, NumVecElts); + CondTy = VectorType::get(ScalarCondTy, NumVecElts); + ++LongVectorCount; + } + // The minimal length of the vector is limited by the real length of vector + // operations performed on the current platform. That's why several final + // reduction opertions are perfomed on the vectors with the same + // architecture-dependent length. + ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) * + ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty, + NumVecElts, Ty); + MinMaxCost += + (NumReduxLevels - LongVectorCount) * + (ConcreteTTI->getCmpSelInstrCost(CmpOpcode, Ty, CondTy, nullptr) + + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, Ty, CondTy, + nullptr)); + // Need 3 extractelement instructions for scalarization + an additional + // scalar select instruction. + return ShuffleCost + MinMaxCost + + 3 * getScalarizationOverhead(Ty, /*Insert=*/false, + /*Extract=*/true) + + ConcreteTTI->getCmpSelInstrCost(Instruction::Select, ScalarTy, + ScalarCondTy, nullptr); + } + unsigned getVectorSplitCost() { return 1; } /// @} @@ -1177,7 +1283,8 @@ public: /// \brief Concrete BasicTTIImpl that can be used if no further customization /// is needed. class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> { - typedef BasicTTIImplBase<BasicTTIImpl> BaseT; + using BaseT = BasicTTIImplBase<BasicTTIImpl>; + friend class BasicTTIImplBase<BasicTTIImpl>; const TargetSubtargetInfo *ST; @@ -1190,6 +1297,6 @@ public: explicit BasicTTIImpl(const TargetMachine *ST, const Function &F); }; -} +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_BASICTTIIMPL_H diff --git a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h index 17c9415a81cb..d9e8206408a7 100644 --- a/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h +++ b/contrib/llvm/include/llvm/CodeGen/CalcSpillWeights.h @@ -1,4 +1,4 @@ -//===---------------- lib/CodeGen/CalcSpillWeights.h ------------*- C++ -*-===// +//===- lib/CodeGen/CalcSpillWeights.h ---------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,7 +7,6 @@ // //===----------------------------------------------------------------------===// - #ifndef LLVM_CODEGEN_CALCSPILLWEIGHTS_H #define LLVM_CODEGEN_CALCSPILLWEIGHTS_H @@ -16,11 +15,12 @@ namespace llvm { - class LiveInterval; - class LiveIntervals; - class MachineBlockFrequencyInfo; - class MachineLoopInfo; - class VirtRegMap; +class LiveInterval; +class LiveIntervals; +class MachineBlockFrequencyInfo; +class MachineFunction; +class MachineLoopInfo; +class VirtRegMap; /// \brief Normalize the spill weight of a live interval /// @@ -32,7 +32,6 @@ namespace llvm { /// per function call. Derived from block frequencies. /// @param Size Size of live interval as returnexd by getSize() /// @param NumInstr Number of instructions using this live interval - /// static inline float normalizeSpillWeight(float UseDefFreq, unsigned Size, unsigned NumInstr) { // The constant 25 instructions is added to avoid depending too much on @@ -47,7 +46,7 @@ namespace llvm { /// spill weight and allocation hint. class VirtRegAuxInfo { public: - typedef float (*NormalizingFn)(float, unsigned, unsigned); + using NormalizingFn = float (*)(float, unsigned, unsigned); private: MachineFunction &MF; @@ -67,6 +66,32 @@ namespace llvm { /// \brief (re)compute li's spill weight and allocation hint. void calculateSpillWeightAndHint(LiveInterval &li); + + /// \brief Compute future expected spill weight of a split artifact of li + /// that will span between start and end slot indexes. + /// \param li The live interval to be split. + /// \param start The expected begining of the split artifact. Instructions + /// before start will not affect the weight. + /// \param end The expected end of the split artifact. Instructions + /// after end will not affect the weight. + /// \return The expected spill weight of the split artifact. Returns + /// negative weight for unspillable li. + float futureWeight(LiveInterval &li, SlotIndex start, SlotIndex end); + + /// \brief Helper function for weight calculations. + /// (Re)compute li's spill weight and allocation hint, or, for non null + /// start and end - compute future expected spill weight of a split + /// artifact of li that will span between start and end slot indexes. + /// \param li The live interval for which to compute the weight. + /// \param start The expected begining of the split artifact. Instructions + /// before start will not affect the weight. Relevant for + /// weight calculation of future split artifact. + /// \param end The expected end of the split artifact. Instructions + /// after end will not affect the weight. Relevant for + /// weight calculation of future split artifact. + /// \return The spill weight. Returns negative weight for unspillable li. + float weightCalcHelper(LiveInterval &li, SlotIndex *start = nullptr, + SlotIndex *end = nullptr); }; /// \brief Compute spill weights and allocation hints for all virtual register @@ -77,6 +102,7 @@ namespace llvm { const MachineBlockFrequencyInfo &MBFI, VirtRegAuxInfo::NormalizingFn norm = normalizeSpillWeight); -} + +} // end namespace llvm #endif // LLVM_CODEGEN_CALCSPILLWEIGHTS_H diff --git a/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h b/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h index 50e464ebb9b8..d30a27328c01 100644 --- a/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h +++ b/contrib/llvm/include/llvm/CodeGen/CallingConvLower.h @@ -1,4 +1,4 @@ -//===-- llvm/CallingConvLower.h - Calling Conventions -----------*- C++ -*-===// +//===- llvm/CallingConvLower.h - Calling Conventions ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -18,11 +18,12 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallingConv.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/Target/TargetCallingConv.h" namespace llvm { + class CCState; class MVT; class TargetMachine; @@ -200,6 +201,7 @@ private: unsigned MaxStackArgAlign; SmallVector<uint32_t, 16> UsedRegs; SmallVector<CCValAssign, 4> PendingLocs; + SmallVector<ISD::ArgFlagsTy, 4> PendingArgFlags; // ByValInfo and SmallVector<ByValInfo, 4> ByValRegs: // @@ -503,10 +505,15 @@ public: } // Get list of pending assignments - SmallVectorImpl<llvm::CCValAssign> &getPendingLocs() { + SmallVectorImpl<CCValAssign> &getPendingLocs() { return PendingLocs; } + // Get a list of argflags for pending assignments. + SmallVectorImpl<ISD::ArgFlagsTy> &getPendingArgFlags() { + return PendingArgFlags; + } + /// Compute the remaining unused register parameters that would be used for /// the given value type. This is useful when varargs are passed in the /// registers that normal prototyped parameters would be passed in, or for @@ -564,8 +571,6 @@ private: void MarkAllocated(unsigned Reg); }; - - } // end namespace llvm -#endif +#endif // LLVM_CODEGEN_CALLINGCONVLOWER_H diff --git a/contrib/llvm/include/llvm/CodeGen/CommandFlags.def b/contrib/llvm/include/llvm/CodeGen/CommandFlags.def new file mode 100644 index 000000000000..fe96033a9c61 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/CommandFlags.def @@ -0,0 +1,366 @@ +//===-- CommandFlags.h - Command Line Flags Interface -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains codegen-specific flags that are shared between different +// command line tools. The tools "llc" and "opt" both use this file to prevent +// flag duplication. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringExtras.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/MC/MCTargetOptionsCommandFlags.def" +#include "llvm/MC/SubtargetFeature.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Host.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include <string> +using namespace llvm; + +static cl::opt<std::string> + MArch("march", + cl::desc("Architecture to generate code for (see --version)")); + +static cl::opt<std::string> + MCPU("mcpu", + cl::desc("Target a specific cpu type (-mcpu=help for details)"), + cl::value_desc("cpu-name"), cl::init("")); + +static cl::list<std::string> + MAttrs("mattr", cl::CommaSeparated, + cl::desc("Target specific attributes (-mattr=help for details)"), + cl::value_desc("a1,+a2,-a3,...")); + +static cl::opt<Reloc::Model> RelocModel( + "relocation-model", cl::desc("Choose relocation model"), + cl::values( + clEnumValN(Reloc::Static, "static", "Non-relocatable code"), + clEnumValN(Reloc::PIC_, "pic", + "Fully relocatable, position independent code"), + clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic", + "Relocatable external references, non-relocatable code"), + clEnumValN(Reloc::ROPI, "ropi", + "Code and read-only data relocatable, accessed PC-relative"), + clEnumValN( + Reloc::RWPI, "rwpi", + "Read-write data relocatable, accessed relative to static base"), + clEnumValN(Reloc::ROPI_RWPI, "ropi-rwpi", + "Combination of ropi and rwpi"))); + +LLVM_ATTRIBUTE_UNUSED static Optional<Reloc::Model> getRelocModel() { + if (RelocModel.getNumOccurrences()) { + Reloc::Model R = RelocModel; + return R; + } + return None; +} + +static cl::opt<ThreadModel::Model> TMModel( + "thread-model", cl::desc("Choose threading model"), + cl::init(ThreadModel::POSIX), + cl::values(clEnumValN(ThreadModel::POSIX, "posix", "POSIX thread model"), + clEnumValN(ThreadModel::Single, "single", + "Single thread model"))); + +static cl::opt<llvm::CodeModel::Model> CMModel( + "code-model", cl::desc("Choose code model"), + cl::values(clEnumValN(CodeModel::Small, "small", "Small code model"), + clEnumValN(CodeModel::Kernel, "kernel", "Kernel code model"), + clEnumValN(CodeModel::Medium, "medium", "Medium code model"), + clEnumValN(CodeModel::Large, "large", "Large code model"))); + +LLVM_ATTRIBUTE_UNUSED static Optional<CodeModel::Model> getCodeModel() { + if (CMModel.getNumOccurrences()) { + CodeModel::Model M = CMModel; + return M; + } + return None; +} + +static cl::opt<llvm::ExceptionHandling> ExceptionModel( + "exception-model", cl::desc("exception model"), + cl::init(ExceptionHandling::None), + cl::values( + clEnumValN(ExceptionHandling::None, "default", + "default exception handling model"), + clEnumValN(ExceptionHandling::DwarfCFI, "dwarf", + "DWARF-like CFI based exception handling"), + clEnumValN(ExceptionHandling::SjLj, "sjlj", "SjLj exception handling"), + clEnumValN(ExceptionHandling::ARM, "arm", "ARM EHABI exceptions"), + clEnumValN(ExceptionHandling::WinEH, "wineh", + "Windows exception model"))); + +static cl::opt<TargetMachine::CodeGenFileType> FileType( + "filetype", cl::init(TargetMachine::CGFT_AssemblyFile), + cl::desc( + "Choose a file type (not all types are supported by all targets):"), + cl::values(clEnumValN(TargetMachine::CGFT_AssemblyFile, "asm", + "Emit an assembly ('.s') file"), + clEnumValN(TargetMachine::CGFT_ObjectFile, "obj", + "Emit a native object ('.o') file"), + clEnumValN(TargetMachine::CGFT_Null, "null", + "Emit nothing, for performance testing"))); + +static cl::opt<bool> + DisableFPElim("disable-fp-elim", + cl::desc("Disable frame pointer elimination optimization"), + cl::init(false)); + +static cl::opt<bool> EnableUnsafeFPMath( + "enable-unsafe-fp-math", + cl::desc("Enable optimizations that may decrease FP precision"), + cl::init(false)); + +static cl::opt<bool> EnableNoInfsFPMath( + "enable-no-infs-fp-math", + cl::desc("Enable FP math optimizations that assume no +-Infs"), + cl::init(false)); + +static cl::opt<bool> EnableNoNaNsFPMath( + "enable-no-nans-fp-math", + cl::desc("Enable FP math optimizations that assume no NaNs"), + cl::init(false)); + +static cl::opt<bool> EnableNoSignedZerosFPMath( + "enable-no-signed-zeros-fp-math", + cl::desc("Enable FP math optimizations that assume " + "the sign of 0 is insignificant"), + cl::init(false)); + +static cl::opt<bool> + EnableNoTrappingFPMath("enable-no-trapping-fp-math", + cl::desc("Enable setting the FP exceptions build " + "attribute not to use exceptions"), + cl::init(false)); + +static cl::opt<llvm::FPDenormal::DenormalMode> DenormalMode( + "denormal-fp-math", + cl::desc("Select which denormal numbers the code is permitted to require"), + cl::init(FPDenormal::IEEE), + cl::values(clEnumValN(FPDenormal::IEEE, "ieee", + "IEEE 754 denormal numbers"), + clEnumValN(FPDenormal::PreserveSign, "preserve-sign", + "the sign of a flushed-to-zero number is preserved " + "in the sign of 0"), + clEnumValN(FPDenormal::PositiveZero, "positive-zero", + "denormals are flushed to positive zero"))); + +static cl::opt<bool> EnableHonorSignDependentRoundingFPMath( + "enable-sign-dependent-rounding-fp-math", cl::Hidden, + cl::desc("Force codegen to assume rounding mode can change dynamically"), + cl::init(false)); + +static cl::opt<llvm::FloatABI::ABIType> FloatABIForCalls( + "float-abi", cl::desc("Choose float ABI type"), cl::init(FloatABI::Default), + cl::values(clEnumValN(FloatABI::Default, "default", + "Target default float ABI type"), + clEnumValN(FloatABI::Soft, "soft", + "Soft float ABI (implied by -soft-float)"), + clEnumValN(FloatABI::Hard, "hard", + "Hard float ABI (uses FP registers)"))); + +static cl::opt<llvm::FPOpFusion::FPOpFusionMode> FuseFPOps( + "fp-contract", cl::desc("Enable aggressive formation of fused FP ops"), + cl::init(FPOpFusion::Standard), + cl::values( + clEnumValN(FPOpFusion::Fast, "fast", "Fuse FP ops whenever profitable"), + clEnumValN(FPOpFusion::Standard, "on", "Only fuse 'blessed' FP ops."), + clEnumValN(FPOpFusion::Strict, "off", + "Only fuse FP ops when the result won't be affected."))); + +static cl::opt<bool> DontPlaceZerosInBSS( + "nozero-initialized-in-bss", + cl::desc("Don't place zero-initialized symbols into bss section"), + cl::init(false)); + +static cl::opt<bool> EnableGuaranteedTailCallOpt( + "tailcallopt", + cl::desc( + "Turn fastcc calls into tail calls by (potentially) changing ABI."), + cl::init(false)); + +static cl::opt<bool> DisableTailCalls("disable-tail-calls", + cl::desc("Never emit tail calls"), + cl::init(false)); + +static cl::opt<bool> StackSymbolOrdering("stack-symbol-ordering", + cl::desc("Order local stack symbols."), + cl::init(true)); + +static cl::opt<unsigned> + OverrideStackAlignment("stack-alignment", + cl::desc("Override default stack alignment"), + cl::init(0)); + +static cl::opt<bool> + StackRealign("stackrealign", + cl::desc("Force align the stack to the minimum alignment"), + cl::init(false)); + +static cl::opt<std::string> TrapFuncName( + "trap-func", cl::Hidden, + cl::desc("Emit a call to trap function rather than a trap instruction"), + cl::init("")); + +static cl::opt<bool> UseCtors("use-ctors", + cl::desc("Use .ctors instead of .init_array."), + cl::init(false)); + +static cl::opt<bool> RelaxELFRelocations( + "relax-elf-relocations", + cl::desc("Emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL on x86-64 ELF"), + cl::init(false)); + +static cl::opt<bool> DataSections("data-sections", + cl::desc("Emit data into separate sections"), + cl::init(false)); + +static cl::opt<bool> + FunctionSections("function-sections", + cl::desc("Emit functions into separate sections"), + cl::init(false)); + +static cl::opt<bool> EmulatedTLS("emulated-tls", + cl::desc("Use emulated TLS model"), + cl::init(false)); + +static cl::opt<bool> + UniqueSectionNames("unique-section-names", + cl::desc("Give unique names to every section"), + cl::init(true)); + +static cl::opt<llvm::EABI> + EABIVersion("meabi", cl::desc("Set EABI type (default depends on triple):"), + cl::init(EABI::Default), + cl::values(clEnumValN(EABI::Default, "default", + "Triple default EABI version"), + clEnumValN(EABI::EABI4, "4", "EABI version 4"), + clEnumValN(EABI::EABI5, "5", "EABI version 5"), + clEnumValN(EABI::GNU, "gnu", "EABI GNU"))); + +static cl::opt<DebuggerKind> DebuggerTuningOpt( + "debugger-tune", cl::desc("Tune debug info for a particular debugger"), + cl::init(DebuggerKind::Default), + cl::values(clEnumValN(DebuggerKind::GDB, "gdb", "gdb"), + clEnumValN(DebuggerKind::LLDB, "lldb", "lldb"), + clEnumValN(DebuggerKind::SCE, "sce", "SCE targets (e.g. PS4)"))); + +static cl::opt<bool> EnableStackSizeSection( + "stack-size-section", + cl::desc("Emit a section containing stack size metadata"), cl::init(false)); + +// Common utility function tightly tied to the options listed here. Initializes +// a TargetOptions object with CodeGen flags and returns it. +static TargetOptions InitTargetOptionsFromCodeGenFlags() { + TargetOptions Options; + Options.AllowFPOpFusion = FuseFPOps; + Options.UnsafeFPMath = EnableUnsafeFPMath; + Options.NoInfsFPMath = EnableNoInfsFPMath; + Options.NoNaNsFPMath = EnableNoNaNsFPMath; + Options.NoSignedZerosFPMath = EnableNoSignedZerosFPMath; + Options.NoTrappingFPMath = EnableNoTrappingFPMath; + Options.FPDenormalMode = DenormalMode; + Options.HonorSignDependentRoundingFPMathOption = + EnableHonorSignDependentRoundingFPMath; + if (FloatABIForCalls != FloatABI::Default) + Options.FloatABIType = FloatABIForCalls; + Options.NoZerosInBSS = DontPlaceZerosInBSS; + Options.GuaranteedTailCallOpt = EnableGuaranteedTailCallOpt; + Options.StackAlignmentOverride = OverrideStackAlignment; + Options.StackSymbolOrdering = StackSymbolOrdering; + Options.UseInitArray = !UseCtors; + Options.RelaxELFRelocations = RelaxELFRelocations; + Options.DataSections = DataSections; + Options.FunctionSections = FunctionSections; + Options.UniqueSectionNames = UniqueSectionNames; + Options.EmulatedTLS = EmulatedTLS; + Options.ExceptionModel = ExceptionModel; + Options.EmitStackSizeSection = EnableStackSizeSection; + + Options.MCOptions = InitMCTargetOptionsFromFlags(); + + Options.ThreadModel = TMModel; + Options.EABIVersion = EABIVersion; + Options.DebuggerTuning = DebuggerTuningOpt; + + return Options; +} + +LLVM_ATTRIBUTE_UNUSED static std::string getCPUStr() { + // If user asked for the 'native' CPU, autodetect here. If autodection fails, + // this will set the CPU to an empty string which tells the target to + // pick a basic default. + if (MCPU == "native") + return sys::getHostCPUName(); + + return MCPU; +} + +LLVM_ATTRIBUTE_UNUSED static std::string getFeaturesStr() { + SubtargetFeatures Features; + + // If user asked for the 'native' CPU, we need to autodetect features. + // This is necessary for x86 where the CPU might not support all the + // features the autodetected CPU name lists in the target. For example, + // not all Sandybridge processors support AVX. + if (MCPU == "native") { + StringMap<bool> HostFeatures; + if (sys::getHostCPUFeatures(HostFeatures)) + for (auto &F : HostFeatures) + Features.AddFeature(F.first(), F.second); + } + + for (unsigned i = 0; i != MAttrs.size(); ++i) + Features.AddFeature(MAttrs[i]); + + return Features.getString(); +} + +/// \brief Set function attributes of functions in Module M based on CPU, +/// Features, and command line flags. +LLVM_ATTRIBUTE_UNUSED static void +setFunctionAttributes(StringRef CPU, StringRef Features, Module &M) { + for (auto &F : M) { + auto &Ctx = F.getContext(); + AttributeList Attrs = F.getAttributes(); + AttrBuilder NewAttrs; + + if (!CPU.empty()) + NewAttrs.addAttribute("target-cpu", CPU); + if (!Features.empty()) + NewAttrs.addAttribute("target-features", Features); + if (DisableFPElim.getNumOccurrences() > 0) + NewAttrs.addAttribute("no-frame-pointer-elim", + DisableFPElim ? "true" : "false"); + if (DisableTailCalls.getNumOccurrences() > 0) + NewAttrs.addAttribute("disable-tail-calls", + toStringRef(DisableTailCalls)); + if (StackRealign) + NewAttrs.addAttribute("stackrealign"); + + if (TrapFuncName.getNumOccurrences() > 0) + for (auto &B : F) + for (auto &I : B) + if (auto *Call = dyn_cast<CallInst>(&I)) + if (const auto *F = Call->getCalledFunction()) + if (F->getIntrinsicID() == Intrinsic::debugtrap || + F->getIntrinsicID() == Intrinsic::trap) + Call->addAttribute( + llvm::AttributeList::FunctionIndex, + Attribute::get(Ctx, "trap-func-name", TrapFuncName)); + + // Let NewAttrs override Attrs. + F.setAttributes( + Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); + } +} diff --git a/contrib/llvm/include/llvm/CodeGen/CommandFlags.h b/contrib/llvm/include/llvm/CodeGen/CommandFlags.h deleted file mode 100644 index 0d898827efc6..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/CommandFlags.h +++ /dev/null @@ -1,382 +0,0 @@ -//===-- CommandFlags.h - Command Line Flags Interface -----------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains codegen-specific flags that are shared between different -// command line tools. The tools "llc" and "opt" both use this file to prevent -// flag duplication. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_COMMANDFLAGS_H -#define LLVM_CODEGEN_COMMANDFLAGS_H - -#include "llvm/ADT/StringExtras.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Module.h" -#include "llvm/MC/MCTargetOptionsCommandFlags.h" -#include "llvm/MC/SubtargetFeature.h" -#include "llvm/Support/CodeGen.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Host.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include <string> -using namespace llvm; - -cl::opt<std::string> -MArch("march", cl::desc("Architecture to generate code for (see --version)")); - -cl::opt<std::string> -MCPU("mcpu", - cl::desc("Target a specific cpu type (-mcpu=help for details)"), - cl::value_desc("cpu-name"), - cl::init("")); - -cl::list<std::string> -MAttrs("mattr", - cl::CommaSeparated, - cl::desc("Target specific attributes (-mattr=help for details)"), - cl::value_desc("a1,+a2,-a3,...")); - -cl::opt<Reloc::Model> RelocModel( - "relocation-model", cl::desc("Choose relocation model"), - cl::values( - clEnumValN(Reloc::Static, "static", "Non-relocatable code"), - clEnumValN(Reloc::PIC_, "pic", - "Fully relocatable, position independent code"), - clEnumValN(Reloc::DynamicNoPIC, "dynamic-no-pic", - "Relocatable external references, non-relocatable code"), - clEnumValN(Reloc::ROPI, "ropi", - "Code and read-only data relocatable, accessed PC-relative"), - clEnumValN(Reloc::RWPI, "rwpi", - "Read-write data relocatable, accessed relative to static base"), - clEnumValN(Reloc::ROPI_RWPI, "ropi-rwpi", - "Combination of ropi and rwpi"))); - -static inline Optional<Reloc::Model> getRelocModel() { - if (RelocModel.getNumOccurrences()) { - Reloc::Model R = RelocModel; - return R; - } - return None; -} - -cl::opt<ThreadModel::Model> -TMModel("thread-model", - cl::desc("Choose threading model"), - cl::init(ThreadModel::POSIX), - cl::values(clEnumValN(ThreadModel::POSIX, "posix", - "POSIX thread model"), - clEnumValN(ThreadModel::Single, "single", - "Single thread model"))); - -cl::opt<llvm::CodeModel::Model> -CMModel("code-model", - cl::desc("Choose code model"), - cl::init(CodeModel::Default), - cl::values(clEnumValN(CodeModel::Default, "default", - "Target default code model"), - clEnumValN(CodeModel::Small, "small", - "Small code model"), - clEnumValN(CodeModel::Kernel, "kernel", - "Kernel code model"), - clEnumValN(CodeModel::Medium, "medium", - "Medium code model"), - clEnumValN(CodeModel::Large, "large", - "Large code model"))); - -cl::opt<llvm::ExceptionHandling> -ExceptionModel("exception-model", - cl::desc("exception model"), - cl::init(ExceptionHandling::None), - cl::values(clEnumValN(ExceptionHandling::None, "default", - "default exception handling model"), - clEnumValN(ExceptionHandling::DwarfCFI, "dwarf", - "DWARF-like CFI based exception handling"), - clEnumValN(ExceptionHandling::SjLj, "sjlj", - "SjLj exception handling"), - clEnumValN(ExceptionHandling::ARM, "arm", - "ARM EHABI exceptions"), - clEnumValN(ExceptionHandling::WinEH, "wineh", - "Windows exception model"))); - -cl::opt<TargetMachine::CodeGenFileType> -FileType("filetype", cl::init(TargetMachine::CGFT_AssemblyFile), - cl::desc("Choose a file type (not all types are supported by all targets):"), - cl::values( - clEnumValN(TargetMachine::CGFT_AssemblyFile, "asm", - "Emit an assembly ('.s') file"), - clEnumValN(TargetMachine::CGFT_ObjectFile, "obj", - "Emit a native object ('.o') file"), - clEnumValN(TargetMachine::CGFT_Null, "null", - "Emit nothing, for performance testing"))); - -cl::opt<bool> -DisableFPElim("disable-fp-elim", - cl::desc("Disable frame pointer elimination optimization"), - cl::init(false)); - -cl::opt<bool> -EnableUnsafeFPMath("enable-unsafe-fp-math", - cl::desc("Enable optimizations that may decrease FP precision"), - cl::init(false)); - -cl::opt<bool> -EnableNoInfsFPMath("enable-no-infs-fp-math", - cl::desc("Enable FP math optimizations that assume no +-Infs"), - cl::init(false)); - -cl::opt<bool> -EnableNoNaNsFPMath("enable-no-nans-fp-math", - cl::desc("Enable FP math optimizations that assume no NaNs"), - cl::init(false)); - -cl::opt<bool> -EnableNoSignedZerosFPMath("enable-no-signed-zeros-fp-math", - cl::desc("Enable FP math optimizations that assume " - "the sign of 0 is insignificant"), - cl::init(false)); - -cl::opt<bool> -EnableNoTrappingFPMath("enable-no-trapping-fp-math", - cl::desc("Enable setting the FP exceptions build " - "attribute not to use exceptions"), - cl::init(false)); - -cl::opt<llvm::FPDenormal::DenormalMode> -DenormalMode("denormal-fp-math", - cl::desc("Select which denormal numbers the code is permitted to require"), - cl::init(FPDenormal::IEEE), - cl::values( - clEnumValN(FPDenormal::IEEE, "ieee", - "IEEE 754 denormal numbers"), - clEnumValN(FPDenormal::PreserveSign, "preserve-sign", - "the sign of a flushed-to-zero number is preserved " - "in the sign of 0"), - clEnumValN(FPDenormal::PositiveZero, "positive-zero", - "denormals are flushed to positive zero"))); - -cl::opt<bool> -EnableHonorSignDependentRoundingFPMath("enable-sign-dependent-rounding-fp-math", - cl::Hidden, - cl::desc("Force codegen to assume rounding mode can change dynamically"), - cl::init(false)); - -cl::opt<llvm::FloatABI::ABIType> -FloatABIForCalls("float-abi", - cl::desc("Choose float ABI type"), - cl::init(FloatABI::Default), - cl::values( - clEnumValN(FloatABI::Default, "default", - "Target default float ABI type"), - clEnumValN(FloatABI::Soft, "soft", - "Soft float ABI (implied by -soft-float)"), - clEnumValN(FloatABI::Hard, "hard", - "Hard float ABI (uses FP registers)"))); - -cl::opt<llvm::FPOpFusion::FPOpFusionMode> -FuseFPOps("fp-contract", - cl::desc("Enable aggressive formation of fused FP ops"), - cl::init(FPOpFusion::Standard), - cl::values( - clEnumValN(FPOpFusion::Fast, "fast", - "Fuse FP ops whenever profitable"), - clEnumValN(FPOpFusion::Standard, "on", - "Only fuse 'blessed' FP ops."), - clEnumValN(FPOpFusion::Strict, "off", - "Only fuse FP ops when the result won't be affected."))); - -cl::opt<bool> -DontPlaceZerosInBSS("nozero-initialized-in-bss", - cl::desc("Don't place zero-initialized symbols into bss section"), - cl::init(false)); - -cl::opt<bool> -EnableGuaranteedTailCallOpt("tailcallopt", - cl::desc("Turn fastcc calls into tail calls by (potentially) changing ABI."), - cl::init(false)); - -cl::opt<bool> -DisableTailCalls("disable-tail-calls", - cl::desc("Never emit tail calls"), - cl::init(false)); - -cl::opt<bool> -StackSymbolOrdering("stack-symbol-ordering", - cl::desc("Order local stack symbols."), - cl::init(true)); - -cl::opt<unsigned> -OverrideStackAlignment("stack-alignment", - cl::desc("Override default stack alignment"), - cl::init(0)); - -cl::opt<bool> -StackRealign("stackrealign", - cl::desc("Force align the stack to the minimum alignment"), - cl::init(false)); - -cl::opt<std::string> -TrapFuncName("trap-func", cl::Hidden, - cl::desc("Emit a call to trap function rather than a trap instruction"), - cl::init("")); - -cl::opt<bool> -UseCtors("use-ctors", - cl::desc("Use .ctors instead of .init_array."), - cl::init(false)); - -cl::opt<bool> RelaxELFRelocations( - "relax-elf-relocations", - cl::desc("Emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL on x86-64 ELF"), - cl::init(false)); - -cl::opt<bool> DataSections("data-sections", - cl::desc("Emit data into separate sections"), - cl::init(false)); - -cl::opt<bool> -FunctionSections("function-sections", - cl::desc("Emit functions into separate sections"), - cl::init(false)); - -cl::opt<bool> EmulatedTLS("emulated-tls", - cl::desc("Use emulated TLS model"), - cl::init(false)); - -cl::opt<bool> UniqueSectionNames("unique-section-names", - cl::desc("Give unique names to every section"), - cl::init(true)); - -cl::opt<llvm::EABI> EABIVersion( - "meabi", cl::desc("Set EABI type (default depends on triple):"), - cl::init(EABI::Default), - cl::values(clEnumValN(EABI::Default, "default", - "Triple default EABI version"), - clEnumValN(EABI::EABI4, "4", "EABI version 4"), - clEnumValN(EABI::EABI5, "5", "EABI version 5"), - clEnumValN(EABI::GNU, "gnu", "EABI GNU"))); - -cl::opt<DebuggerKind> -DebuggerTuningOpt("debugger-tune", - cl::desc("Tune debug info for a particular debugger"), - cl::init(DebuggerKind::Default), - cl::values( - clEnumValN(DebuggerKind::GDB, "gdb", "gdb"), - clEnumValN(DebuggerKind::LLDB, "lldb", "lldb"), - clEnumValN(DebuggerKind::SCE, "sce", - "SCE targets (e.g. PS4)"))); - -// Common utility function tightly tied to the options listed here. Initializes -// a TargetOptions object with CodeGen flags and returns it. -static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { - TargetOptions Options; - Options.AllowFPOpFusion = FuseFPOps; - Options.UnsafeFPMath = EnableUnsafeFPMath; - Options.NoInfsFPMath = EnableNoInfsFPMath; - Options.NoNaNsFPMath = EnableNoNaNsFPMath; - Options.NoSignedZerosFPMath = EnableNoSignedZerosFPMath; - Options.NoTrappingFPMath = EnableNoTrappingFPMath; - Options.FPDenormalMode = DenormalMode; - Options.HonorSignDependentRoundingFPMathOption = - EnableHonorSignDependentRoundingFPMath; - if (FloatABIForCalls != FloatABI::Default) - Options.FloatABIType = FloatABIForCalls; - Options.NoZerosInBSS = DontPlaceZerosInBSS; - Options.GuaranteedTailCallOpt = EnableGuaranteedTailCallOpt; - Options.StackAlignmentOverride = OverrideStackAlignment; - Options.StackSymbolOrdering = StackSymbolOrdering; - Options.UseInitArray = !UseCtors; - Options.RelaxELFRelocations = RelaxELFRelocations; - Options.DataSections = DataSections; - Options.FunctionSections = FunctionSections; - Options.UniqueSectionNames = UniqueSectionNames; - Options.EmulatedTLS = EmulatedTLS; - Options.ExceptionModel = ExceptionModel; - - Options.MCOptions = InitMCTargetOptionsFromFlags(); - - Options.ThreadModel = TMModel; - Options.EABIVersion = EABIVersion; - Options.DebuggerTuning = DebuggerTuningOpt; - - return Options; -} - -static inline std::string getCPUStr() { - // If user asked for the 'native' CPU, autodetect here. If autodection fails, - // this will set the CPU to an empty string which tells the target to - // pick a basic default. - if (MCPU == "native") - return sys::getHostCPUName(); - - return MCPU; -} - -static inline std::string getFeaturesStr() { - SubtargetFeatures Features; - - // If user asked for the 'native' CPU, we need to autodetect features. - // This is necessary for x86 where the CPU might not support all the - // features the autodetected CPU name lists in the target. For example, - // not all Sandybridge processors support AVX. - if (MCPU == "native") { - StringMap<bool> HostFeatures; - if (sys::getHostCPUFeatures(HostFeatures)) - for (auto &F : HostFeatures) - Features.AddFeature(F.first(), F.second); - } - - for (unsigned i = 0; i != MAttrs.size(); ++i) - Features.AddFeature(MAttrs[i]); - - return Features.getString(); -} - -/// \brief Set function attributes of functions in Module M based on CPU, -/// Features, and command line flags. -static inline void setFunctionAttributes(StringRef CPU, StringRef Features, - Module &M) { - for (auto &F : M) { - auto &Ctx = F.getContext(); - AttributeList Attrs = F.getAttributes(); - AttrBuilder NewAttrs; - - if (!CPU.empty()) - NewAttrs.addAttribute("target-cpu", CPU); - if (!Features.empty()) - NewAttrs.addAttribute("target-features", Features); - if (DisableFPElim.getNumOccurrences() > 0) - NewAttrs.addAttribute("no-frame-pointer-elim", - DisableFPElim ? "true" : "false"); - if (DisableTailCalls.getNumOccurrences() > 0) - NewAttrs.addAttribute("disable-tail-calls", - toStringRef(DisableTailCalls)); - if (StackRealign) - NewAttrs.addAttribute("stackrealign"); - - if (TrapFuncName.getNumOccurrences() > 0) - for (auto &B : F) - for (auto &I : B) - if (auto *Call = dyn_cast<CallInst>(&I)) - if (const auto *F = Call->getCalledFunction()) - if (F->getIntrinsicID() == Intrinsic::debugtrap || - F->getIntrinsicID() == Intrinsic::trap) - Call->addAttribute( - llvm::AttributeList::FunctionIndex, - Attribute::get(Ctx, "trap-func-name", TrapFuncName)); - - // Let NewAttrs override Attrs. - F.setAttributes( - Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); - } -} - -#endif diff --git a/contrib/llvm/include/llvm/Target/CostTable.h b/contrib/llvm/include/llvm/CodeGen/CostTable.h index b7d9240a91f5..5a6368c5a0f8 100644 --- a/contrib/llvm/include/llvm/Target/CostTable.h +++ b/contrib/llvm/include/llvm/CodeGen/CostTable.h @@ -12,8 +12,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_COSTTABLE_H_ -#define LLVM_TARGET_COSTTABLE_H_ +#ifndef LLVM_CODEGEN_COSTTABLE_H_ +#define LLVM_CODEGEN_COSTTABLE_H_ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" @@ -66,5 +66,4 @@ ConvertCostTableLookup(ArrayRef<TypeConversionCostTblEntry> Tbl, } // namespace llvm - -#endif /* LLVM_TARGET_COSTTABLE_H_ */ +#endif /* LLVM_CODEGEN_COSTTABLE_H_ */ diff --git a/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h b/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h index 77c37ac7abea..d3aabe22f216 100644 --- a/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h +++ b/contrib/llvm/include/llvm/CodeGen/DFAPacketizer.h @@ -208,6 +208,13 @@ public: // Add a DAG mutation to be done before the packetization begins. void addMutation(std::unique_ptr<ScheduleDAGMutation> Mutation); + + bool alias(const MachineInstr &MI1, const MachineInstr &MI2, + bool UseTBAA = true) const; + +private: + bool alias(const MachineMemOperand &Op1, const MachineMemOperand &Op2, + bool UseTBAA = true) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm/include/llvm/CodeGen/FastISel.h index 74e4179e73e9..85bb826dcb8c 100644 --- a/contrib/llvm/include/llvm/CodeGen/FastISel.h +++ b/contrib/llvm/include/llvm/CodeGen/FastISel.h @@ -20,6 +20,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineValueType.h" +#include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" @@ -27,7 +28,6 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/Target/TargetLowering.h" #include <algorithm> #include <cstdint> #include <utility> diff --git a/contrib/llvm/include/llvm/CodeGen/FaultMaps.h b/contrib/llvm/include/llvm/CodeGen/FaultMaps.h index 98ff526dfe94..55e25c9823b1 100644 --- a/contrib/llvm/include/llvm/CodeGen/FaultMaps.h +++ b/contrib/llvm/include/llvm/CodeGen/FaultMaps.h @@ -39,6 +39,9 @@ public: void recordFaultingOp(FaultKind FaultTy, const MCSymbol *HandlerLabel); void serializeToFaultMapSection(); + void reset() { + FunctionInfos.clear(); + } private: static const char *WFMP; diff --git a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index f32a58915118..3b39d87ffb4a 100644 --- a/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -23,11 +23,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/Support/KnownBits.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <utility> #include <vector> diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index e7ce1946889e..ba84d76de164 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -18,10 +18,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/MachineValueType.h" +#include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Target/TargetCallingConv.h" #include <cstdint> #include <functional> diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelAccessor.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelAccessor.h deleted file mode 100644 index 8dea38059ea4..000000000000 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelAccessor.h +++ /dev/null @@ -1,39 +0,0 @@ -//===-- GISelAccessor.h - GISel Accessor ------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -/// This file declares the API to access the various APIs related -/// to GlobalISel. -// -//===----------------------------------------------------------------------===/ - -#ifndef LLVM_CODEGEN_GLOBALISEL_GISELACCESSOR_H -#define LLVM_CODEGEN_GLOBALISEL_GISELACCESSOR_H - -namespace llvm { -class CallLowering; -class InstructionSelector; -class LegalizerInfo; -class RegisterBankInfo; - -/// The goal of this helper class is to gather the accessor to all -/// the APIs related to GlobalISel. -/// It should be derived to feature an actual accessor to the GISel APIs. -/// The reason why this is not simply done into the subtarget is to avoid -/// spreading ifdefs around. -struct GISelAccessor { - virtual ~GISelAccessor() {} - virtual const CallLowering *getCallLowering() const { return nullptr;} - virtual const InstructionSelector *getInstructionSelector() const { - return nullptr; - } - virtual const LegalizerInfo *getLegalizerInfo() const { return nullptr; } - virtual const RegisterBankInfo *getRegBankInfo() const { return nullptr;} -}; -} // End namespace llvm; -#endif diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h new file mode 100644 index 000000000000..167905dc9aa1 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -0,0 +1,69 @@ +//===- GISelWorkList.h - Worklist for GISel passes ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_GISEL_WORKLIST_H +#define LLVM_GISEL_WORKLIST_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" + +namespace llvm { + +class MachineInstr; + +// Worklist which mostly works similar to InstCombineWorkList, but on MachineInstrs. +// The main difference with something like a SetVector is that erasing an element doesn't +// move all elements over one place - instead just nulls out the element of the vector. +// FIXME: Does it make sense to factor out common code with the instcombinerWorkList? +template<unsigned N> +class GISelWorkList { + SmallVector<MachineInstr*, N> Worklist; + DenseMap<MachineInstr*, unsigned> WorklistMap; + +public: + GISelWorkList() = default; + + bool empty() const { return WorklistMap.empty(); } + + unsigned size() const { return WorklistMap.size(); } + + /// Add - Add the specified instruction to the worklist if it isn't already + /// in it. + void insert(MachineInstr *I) { + if (WorklistMap.try_emplace(I, Worklist.size()).second) { + Worklist.push_back(I); + } + } + + /// Remove - remove I from the worklist if it exists. + void remove(MachineInstr *I) { + auto It = WorklistMap.find(I); + if (It == WorklistMap.end()) return; // Not in worklist. + + // Don't bother moving everything down, just null out the slot. + Worklist[It->second] = nullptr; + + WorklistMap.erase(It); + } + + MachineInstr *pop_back_val() { + MachineInstr *I; + do { + I = Worklist.pop_back_val(); + } while(!I); + assert(I && "Pop back on empty worklist"); + WorklistMap.erase(I); + return I; + } +}; + +} // end namespace llvm. + +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 1060d8fd667e..e599a1b179ec 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -16,7 +16,10 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/CodeGenCoverage.h" #include <bitset> #include <cstddef> #include <cstdint> @@ -26,9 +29,12 @@ namespace llvm { +class APInt; +class APFloat; class LLT; class MachineInstr; class MachineInstrBuilder; +class MachineFunction; class MachineOperand; class MachineRegisterInfo; class RegisterBankInfo; @@ -63,6 +69,18 @@ public: }; enum { + /// Begin a try-block to attempt a match and jump to OnFail if it is + /// unsuccessful. + /// - OnFail - The MatchTable entry at which to resume if the match fails. + /// + /// FIXME: This ought to take an argument indicating the number of try-blocks + /// to exit on failure. It's usually one but the last match attempt of + /// a block will need more. The (implemented) alternative is to tack a + /// GIM_Reject on the end of each try-block which is simpler but + /// requires an extra opcode and iteration in the interpreter on each + /// failed match. + GIM_Try, + /// Record the specified instruction /// - NewInsnID - Instruction ID to define /// - InsnID - Instruction ID @@ -81,12 +99,35 @@ enum { /// - InsnID - Instruction ID /// - Expected number of operands GIM_CheckNumOperands, + /// Check an immediate predicate on the specified instruction + /// - InsnID - Instruction ID + /// - The predicate to test + GIM_CheckI64ImmPredicate, + /// Check an immediate predicate on the specified instruction via an APInt. + /// - InsnID - Instruction ID + /// - The predicate to test + GIM_CheckAPIntImmPredicate, + /// Check a floating point immediate predicate on the specified instruction. + /// - InsnID - Instruction ID + /// - The predicate to test + GIM_CheckAPFloatImmPredicate, + /// Check a memory operation has the specified atomic ordering. + /// - InsnID - Instruction ID + /// - Ordering - The AtomicOrdering value + GIM_CheckAtomicOrdering, + GIM_CheckAtomicOrderingOrStrongerThan, + GIM_CheckAtomicOrderingWeakerThan, /// Check the type for the specified operand /// - InsnID - Instruction ID /// - OpIdx - Operand index /// - Expected type GIM_CheckType, + /// Check the type of a pointer to any address space. + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - SizeInBits - The size of the pointer value in bits. + GIM_CheckPointerToAny, /// Check the register bank for the specified operand /// - InsnID - Instruction ID /// - OpIdx - Operand index @@ -124,6 +165,17 @@ enum { /// - InsnID - Instruction ID GIM_CheckIsSafeToFold, + /// Check the specified operands are identical. + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - OtherInsnID - Other instruction ID + /// - OtherOpIdx - Other operand index + GIM_CheckIsSameOperand, + + /// Fail the current try-block, or completely fail to match if there is no + /// current try-block. + GIM_Reject, + //=== Renderers === /// Mutate an instruction @@ -141,6 +193,13 @@ enum { /// - OldInsnID - Instruction ID to copy from /// - OpIdx - The operand to copy GIR_Copy, + /// Copy an operand to the specified instruction or add a zero register if the + /// operand is a zero immediate. + /// - NewInsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to copy from + /// - OpIdx - The operand to copy + /// - ZeroReg - The zero register to use + GIR_CopyOrAddZeroReg, /// Copy an operand to the specified instruction /// - NewInsnID - Instruction ID to modify /// - OldInsnID - Instruction ID to copy from @@ -159,6 +218,10 @@ enum { /// - InsnID - Instruction ID to modify /// - RegNum - The register to add GIR_AddRegister, + /// Add a a temporary register to the specified instruction + /// - InsnID - Instruction ID to modify + /// - TempRegID - The temporary register ID to add + GIR_AddTempRegister, /// Add an immediate to the specified instruction /// - InsnID - Instruction ID to modify /// - Imm - The immediate to add @@ -167,6 +230,17 @@ enum { /// - InsnID - Instruction ID to modify /// - RendererID - The renderer to call GIR_ComplexRenderer, + /// Render sub-operands of complex operands to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RendererID - The renderer to call + /// - RenderOpID - The suboperand to render. + GIR_ComplexSubOperandRenderer, + + /// Render a G_CONSTANT operator as a sign-extended immediate. + /// - NewInsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to copy from + /// The operand index is implicitly 1. + GIR_CopyConstantAsSImm, /// Constrain an instruction operand to a register class. /// - InsnID - Instruction ID to modify @@ -179,18 +253,39 @@ enum { GIR_ConstrainSelectedInstOperands, /// Merge all memory operands into instruction. /// - InsnID - Instruction ID to modify + /// - MergeInsnID... - One or more Instruction ID to merge into the result. + /// - GIU_MergeMemOperands_EndOfList - Terminates the list of instructions to + /// merge. GIR_MergeMemOperands, /// Erase from parent. /// - InsnID - Instruction ID to erase GIR_EraseFromParent, + /// Create a new temporary register that's not constrained. + /// - TempRegID - The temporary register ID to initialize. + /// - Expected type + GIR_MakeTempReg, /// A successful emission GIR_Done, + + /// Increment the rule coverage counter. + /// - RuleID - The ID of the rule that was covered. + GIR_Coverage, +}; + +enum { + /// Indicates the end of the variable-length MergeInsnID list in a + /// GIR_MergeMemOperands opcode. + GIU_MergeMemOperands_EndOfList = -1, }; /// Provides the logic to select generic machine instructions. class InstructionSelector { public: + using I64ImmediatePredicateFn = bool (*)(int64_t); + using APIntImmediatePredicateFn = bool (*)(const APInt &); + using APFloatImmediatePredicateFn = bool (*)(const APFloat &); + virtual ~InstructionSelector() = default; /// Select the (possibly generic) instruction \p I to only use target-specific @@ -203,17 +298,18 @@ public: /// if returns true: /// for I in all mutated/inserted instructions: /// !isPreISelGenericOpcode(I.getOpcode()) - /// - virtual bool select(MachineInstr &I) const = 0; + virtual bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const = 0; protected: - using ComplexRendererFn = std::function<void(MachineInstrBuilder &)>; + using ComplexRendererFns = + Optional<SmallVector<std::function<void(MachineInstrBuilder &)>, 4>>; using RecordedMIVector = SmallVector<MachineInstr *, 4>; using NewMIVector = SmallVector<MachineInstrBuilder, 4>; struct MatcherState { - std::vector<ComplexRendererFn> Renderers; + std::vector<ComplexRendererFns::value_type> Renderers; RecordedMIVector MIs; + DenseMap<unsigned, unsigned> TempRegisters; MatcherState(unsigned MaxRenderers); }; @@ -223,7 +319,10 @@ public: struct MatcherInfoTy { const LLT *TypeObjects; const PredicateBitset *FeatureBitsets; - const std::vector<ComplexMatcherMemFn> ComplexPredicates; + const I64ImmediatePredicateFn *I64ImmPredicateFns; + const APIntImmediatePredicateFn *APIntImmPredicateFns; + const APFloatImmediatePredicateFn *APFloatImmPredicateFns; + const ComplexMatcherMemFn *ComplexPredicates; }; protected: @@ -238,8 +337,8 @@ protected: const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI, - const PredicateBitset &AvailableFeatures) const; + const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, + CodeGenCoverage &CoverageInfo) const; /// Constrain a register operand of an instruction \p I to a specified /// register class. This could involve inserting COPYs before (for uses) or @@ -268,7 +367,16 @@ protected: bool isOperandImmEqual(const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI) const; - bool isObviouslySafeToFold(MachineInstr &MI) const; + /// Return true if the specified operand is a G_GEP with a G_CONSTANT on the + /// right-hand side. GlobalISel's separation of pointer and integer types + /// means that we don't need to worry about G_OR with equivalent semantics. + bool isBaseWithConstantOffset(const MachineOperand &Root, + const MachineRegisterInfo &MRI) const; + + /// Return true if MI can obviously be folded into IntoMI. + /// MI and IntoMI do not need to be in the same basic blocks, but MI must + /// preceed IntoMI. + bool isObviouslySafeToFold(MachineInstr &MI, MachineInstr &IntoMI) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 98b6b859b9e2..ac2c055ab145 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -1,4 +1,4 @@ -//==-- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h ---------*- C++ -*-==// +//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,7 +16,32 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <cstddef> +#include <cstdint> + namespace llvm { + +/// GlobalISel PatFrag Predicates +enum { + GIPFP_I64_Invalid = 0, + GIPFP_APInt_Invalid = 0, + GIPFP_APFloat_Invalid = 0, +}; + template <class TgtInstructionSelector, class PredicateBitset, class ComplexMatcherMemFn> bool InstructionSelector::executeMatchTable( @@ -24,306 +49,687 @@ bool InstructionSelector::executeMatchTable( const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI, - const PredicateBitset &AvailableFeatures) const { - const int64_t *Command = MatchTable; + const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, + CodeGenCoverage &CoverageInfo) const { + uint64_t CurrentIdx = 0; + SmallVector<uint64_t, 8> OnFailResumeAt; + + enum RejectAction { RejectAndGiveUp, RejectAndResume }; + auto handleReject = [&]() -> RejectAction { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Rejected\n"); + if (OnFailResumeAt.empty()) + return RejectAndGiveUp; + CurrentIdx = OnFailResumeAt.back(); + OnFailResumeAt.pop_back(); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" + << OnFailResumeAt.size() << " try-blocks remain)\n"); + return RejectAndResume; + }; + while (true) { - switch (*Command++) { + assert(CurrentIdx != ~0u && "Invalid MatchTable index"); + switch (MatchTable[CurrentIdx++]) { + case GIM_Try: { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Begin try-block\n"); + OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); + break; + } + case GIM_RecordInsn: { - int64_t NewInsnID = *Command++; - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; // As an optimisation we require that MIs[0] is always the root. Refuse // any attempt to modify it. assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); - (void)NewInsnID; MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); if (!MO.isReg()) { - DEBUG(dbgs() << "Rejected (not a register)\n"); - return false; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Not a register\n"); + if (handleReject() == RejectAndGiveUp) + return false; + break; } if (TRI.isPhysicalRegister(MO.getReg())) { - DEBUG(dbgs() << "Rejected (is a physical register)\n"); - return false; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Is a physical register\n"); + if (handleReject() == RejectAndGiveUp) + return false; + break; } - assert((size_t)NewInsnID == State.MIs.size() && - "Expected to store MIs in order"); - State.MIs.push_back(MRI.getVRegDef(MO.getReg())); - DEBUG(dbgs() << "MIs[" << NewInsnID << "] = GIM_RecordInsn(" << InsnID - << ", " << OpIdx << ")\n"); + MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); + if ((size_t)NewInsnID < State.MIs.size()) + State.MIs[NewInsnID] = NewMI; + else { + assert((size_t)NewInsnID == State.MIs.size() && + "Expected to store MIs in order"); + State.MIs.push_back(NewMI); + } + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": MIs[" << NewInsnID + << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx + << ")\n"); break; } case GIM_CheckFeatures: { - int64_t ExpectedBitsetID = *Command++; - DEBUG(dbgs() << "GIM_CheckFeatures(ExpectedBitsetID=" << ExpectedBitsetID - << ")\n"); + int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckFeatures(ExpectedBitsetID=" + << ExpectedBitsetID << ")\n"); if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) != MatcherInfo.FeatureBitsets[ExpectedBitsetID]) { - DEBUG(dbgs() << "Rejected\n"); - return false; + if (handleReject() == RejectAndGiveUp) + return false; } break; } case GIM_CheckOpcode: { - int64_t InsnID = *Command++; - int64_t Expected = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Expected = MatchTable[CurrentIdx++]; unsigned Opcode = State.MIs[InsnID]->getOpcode(); - DEBUG(dbgs() << "GIM_CheckOpcode(MIs[" << InsnID << "], ExpectedOpcode=" - << Expected << ") // Got=" << Opcode << "\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID + << "], ExpectedOpcode=" << Expected + << ") // Got=" << Opcode << "\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (Opcode != Expected) - return false; + if (Opcode != Expected) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckNumOperands: { - int64_t InsnID = *Command++; - int64_t Expected = *Command++; - DEBUG(dbgs() << "GIM_CheckNumOperands(MIs[" << InsnID - << "], Expected=" << Expected << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Expected = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" + << InsnID << "], Expected=" << Expected << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (State.MIs[InsnID]->getNumOperands() != Expected) - return false; + if (State.MIs[InsnID]->getNumOperands() != Expected) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckI64ImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && + "Expected G_CONSTANT"); + assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); + int64_t Value = 0; + if (State.MIs[InsnID]->getOperand(1).isCImm()) + Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); + else if (State.MIs[InsnID]->getOperand(1).isImm()) + Value = State.MIs[InsnID]->getOperand(1).getImm(); + else + llvm_unreachable("Expected Imm or CImm operand"); + + if (!MatcherInfo.I64ImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckAPIntImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() && "Expected G_CONSTANT"); + assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); + APInt Value; + if (State.MIs[InsnID]->getOperand(1).isCImm()) + Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); + else + llvm_unreachable("Expected Imm or CImm operand"); + + if (!MatcherInfo.APIntImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAPFloatImmPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && + "Expected G_FCONSTANT"); + assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); + assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); + APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); + + if (!MatcherInfo.APFloatImmPredicateFns[Predicate](Value)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrdering: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (MMO->getOrdering() != Ordering) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrderingOrStrongerThan: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + case GIM_CheckAtomicOrderingWeakerThan: { + int64_t InsnID = MatchTable[CurrentIdx++]; + AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" + << InsnID << "], " << (uint64_t)Ordering << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (!isStrongerThan(Ordering, MMO->getOrdering())) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckType: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t TypeID = *Command++; - DEBUG(dbgs() << "GIM_CheckType(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "), TypeID=" << TypeID << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t TypeID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID + << "]->getOperand(" << OpIdx + << "), TypeID=" << TypeID << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) != - MatcherInfo.TypeObjects[TypeID]) - return false; + MatcherInfo.TypeObjects[TypeID]) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckPointerToAny: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t SizeInBits = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), SizeInBits=" << SizeInBits << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + // iPTR must be looked up in the target. + if (SizeInBits == 0) { + MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); + SizeInBits = MF->getDataLayout().getPointerSizeInBits(0); + } + + assert(SizeInBits != 0 && "Pointer size must be known"); + + const LLT &Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); + if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } case GIM_CheckRegBankForClass: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RCEnum = *Command++; - DEBUG(dbgs() << "GIM_CheckRegBankForClass(MIs[" << InsnID - << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum - << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RCEnum = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), RCEnum=" << RCEnum << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != - RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, TRI)) - return false; + RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, + TRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckComplexPattern: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RendererID = *Command++; - int64_t ComplexPredicateID = *Command++; - DEBUG(dbgs() << "State.Renderers[" << RendererID - << "] = GIM_CheckComplexPattern(MIs[" << InsnID - << "]->getOperand(" << OpIdx - << "), ComplexPredicateID=" << ComplexPredicateID << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": State.Renderers[" << RendererID + << "] = GIM_CheckComplexPattern(MIs[" << InsnID + << "]->getOperand(" << OpIdx + << "), ComplexPredicateID=" << ComplexPredicateID + << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); // FIXME: Use std::invoke() when it's available. - if (!(State.Renderers[RendererID] = - (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( - State.MIs[InsnID]->getOperand(OpIdx)))) - return false; + ComplexRendererFns Renderer = + (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( + State.MIs[InsnID]->getOperand(OpIdx)); + if (Renderer.hasValue()) + State.Renderers[RendererID] = Renderer.getValue(); + else + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckConstantInt: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckConstantInt(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, MRI)) - return false; + + // isOperandImmEqual() will sign-extend to 64-bits, so should we. + LLT Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); + Value = SignExtend64(Value, Ty.getSizeInBits()); + + if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, + MRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckLiteralInt: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckLiteralInt(MIs[" << InsnID << "]->getOperand(" << OpIdx - << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); - if (!OM.isCImm() || !OM.getCImm()->equalsInt(Value)) - return false; + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_CheckIntrinsicID: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t Value = *Command++; - DEBUG(dbgs() << "GIM_CheckIntrinsicID(MIs[" << InsnID << "]->getOperand(" << OpIdx - << "), Value=" << Value << ")\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t Value = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - MachineOperand &OM = State.MIs[InsnID]->getOperand(OpIdx); - if (!OM.isIntrinsicID() || OM.getIntrinsicID() != Value) - return false; + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) + if (handleReject() == RejectAndGiveUp) + return false; break; } + case GIM_CheckIsMBB: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - DEBUG(dbgs() << "GIM_CheckIsMBB(MIs[" << InsnID << "]->getOperand(" - << OpIdx << "))\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID + << "]->getOperand(" << OpIdx << "))\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) - return false; + if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } case GIM_CheckIsSafeToFold: { - int64_t InsnID = *Command++; - DEBUG(dbgs() << "GIM_CheckIsSafeToFold(MIs[" << InsnID << "])\n"); + int64_t InsnID = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" + << InsnID << "])\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!isObviouslySafeToFold(*State.MIs[InsnID])) - return false; + if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_CheckIsSameOperand: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t OtherInsnID = MatchTable[CurrentIdx++]; + int64_t OtherOpIdx = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" + << InsnID << "][" << OpIdx << "], MIs[" + << OtherInsnID << "][" << OtherOpIdx << "])\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); + if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( + State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { + if (handleReject() == RejectAndGiveUp) + return false; + } break; } + case GIM_Reject: + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_Reject"); + if (handleReject() == RejectAndGiveUp) + return false; + break; case GIR_MutateOpcode: { - int64_t OldInsnID = *Command++; - int64_t NewInsnID = *Command++; - int64_t NewOpcode = *Command++; - assert((size_t)NewInsnID == OutMIs.size() && - "Expected to store MIs in order"); - OutMIs.push_back( - MachineInstrBuilder(*State.MIs[OldInsnID]->getParent()->getParent(), - State.MIs[OldInsnID])); + int64_t OldInsnID = MatchTable[CurrentIdx++]; + uint64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t NewOpcode = MatchTable[CurrentIdx++]; + if (NewInsnID >= OutMIs.size()) + OutMIs.resize(NewInsnID + 1); + + OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), + State.MIs[OldInsnID]); OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); - DEBUG(dbgs() << "GIR_MutateOpcode(OutMIs[" << NewInsnID << "], MIs[" - << OldInsnID << "], " << NewOpcode << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << NewOpcode << ")\n"); break; } + case GIR_BuildMI: { - int64_t InsnID = *Command++; - int64_t Opcode = *Command++; - assert((size_t)InsnID == OutMIs.size() && - "Expected to store MIs in order"); - (void)InsnID; - OutMIs.push_back(BuildMI(*State.MIs[0]->getParent(), State.MIs[0], - State.MIs[0]->getDebugLoc(), TII.get(Opcode))); - DEBUG(dbgs() << "GIR_BuildMI(OutMIs[" << InsnID << "], " << Opcode - << ")\n"); + uint64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t Opcode = MatchTable[CurrentIdx++]; + if (NewInsnID >= OutMIs.size()) + OutMIs.resize(NewInsnID + 1); + + OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], + State.MIs[0]->getDebugLoc(), TII.get(Opcode)); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" + << NewInsnID << "], " << Opcode << ")\n"); break; } case GIR_Copy: { - int64_t NewInsnID = *Command++; - int64_t OldInsnID = *Command++; - int64_t OpIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); - DEBUG(dbgs() << "GIR_Copy(OutMIs[" << NewInsnID << "], MIs[" << OldInsnID - << "], " << OpIdx << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID + << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); break; } + + case GIR_CopyOrAddZeroReg: { + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t ZeroReg = MatchTable[CurrentIdx++]; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); + if (isOperandImmEqual(MO, 0, MRI)) + OutMIs[NewInsnID].addReg(ZeroReg); + else + OutMIs[NewInsnID].add(MO); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << OpIdx << ", " << ZeroReg << ")\n"); + break; + } + case GIR_CopySubReg: { - int64_t NewInsnID = *Command++; - int64_t OldInsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t SubRegIdx = *Command++; + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t SubRegIdx = MatchTable[CurrentIdx++]; assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 0, SubRegIdx); - DEBUG(dbgs() << "GIR_CopySubReg(OutMIs[" << NewInsnID << "], MIs[" - << OldInsnID << "], " << OpIdx << ", " << SubRegIdx - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "], " + << OpIdx << ", " << SubRegIdx << ")\n"); break; } + case GIR_AddImplicitDef: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addDef(RegNum, RegState::Implicit); - DEBUG(dbgs() << "GIR_AddImplicitDef(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); break; } + case GIR_AddImplicitUse: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addUse(RegNum, RegState::Implicit); - DEBUG(dbgs() << "GIR_AddImplicitUse(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); break; } + case GIR_AddRegister: { - int64_t InsnID = *Command++; - int64_t RegNum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RegNum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addReg(RegNum); - DEBUG(dbgs() << "GIR_AddRegister(OutMIs[" << InsnID << "], " << RegNum - << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" + << InsnID << "], " << RegNum << ")\n"); + break; + } + + case GIR_AddTempRegister: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t TempRegID = MatchTable[CurrentIdx++]; + uint64_t TempRegFlags = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" + << InsnID << "], TempRegisters[" << TempRegID + << "], " << TempRegFlags << ")\n"); break; } + case GIR_AddImm: { - int64_t InsnID = *Command++; - int64_t Imm = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Imm = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); OutMIs[InsnID].addImm(Imm); - DEBUG(dbgs() << "GIR_AddImm(OutMIs[" << InsnID << "], " << Imm << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID + << "], " << Imm << ")\n"); break; } + case GIR_ComplexRenderer: { - int64_t InsnID = *Command++; - int64_t RendererID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + for (const auto &RenderOpFn : State.Renderers[RendererID]) + RenderOpFn(OutMIs[InsnID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" + << InsnID << "], " << RendererID << ")\n"); + break; + } + case GIR_ComplexSubOperandRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + int64_t RenderOpID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - State.Renderers[RendererID](OutMIs[InsnID]); - DEBUG(dbgs() << "GIR_ComplexRenderer(OutMIs[" << InsnID << "], " - << RendererID << ")\n"); + State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIR_ComplexSubOperandRenderer(OutMIs[" + << InsnID << "], " << RendererID << ", " + << RenderOpID << ")\n"); + break; + } + + case GIR_CopyConstantAsSImm: { + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { + OutMIs[NewInsnID].addImm( + State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); + } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) + OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); + else + llvm_unreachable("Expected Imm or CImm operand"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "])\n"); break; } case GIR_ConstrainOperandRC: { - int64_t InsnID = *Command++; - int64_t OpIdx = *Command++; - int64_t RCEnum = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RCEnum = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, *TRI.getRegClass(RCEnum), TII, TRI, RBI); - DEBUG(dbgs() << "GIR_ConstrainOperandRC(OutMIs[" << InsnID << "], " - << OpIdx << ", " << RCEnum << ")\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" + << InsnID << "], " << OpIdx << ", " << RCEnum + << ")\n"); break; } + case GIR_ConstrainSelectedInstOperands: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, RBI); - DEBUG(dbgs() << "GIR_ConstrainSelectedInstOperands(OutMIs[" << InsnID - << "])\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIR_ConstrainSelectedInstOperands(OutMIs[" + << InsnID << "])\n"); break; } + case GIR_MergeMemOperands: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - for (const auto *FromMI : State.MIs) - for (const auto &MMO : FromMI->memoperands()) + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" + << InsnID << "]"); + int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; + while ((MergeInsnID = MatchTable[CurrentIdx++]) != + GIU_MergeMemOperands_EndOfList) { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << ", MIs[" << MergeInsnID << "]"); + for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) OutMIs[InsnID].addMemOperand(MMO); - DEBUG(dbgs() << "GIR_MergeMemOperands(OutMIs[" << InsnID << "])\n"); + } + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); break; } + case GIR_EraseFromParent: { - int64_t InsnID = *Command++; + int64_t InsnID = MatchTable[CurrentIdx++]; assert(State.MIs[InsnID] && "Attempted to erase an undefined instruction"); State.MIs[InsnID]->eraseFromParent(); - DEBUG(dbgs() << "GIR_EraseFromParent(MIs[" << InsnID << "])\n"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" + << InsnID << "])\n"); + break; + } + + case GIR_MakeTempReg: { + int64_t TempRegID = MatchTable[CurrentIdx++]; + int64_t TypeID = MatchTable[CurrentIdx++]; + + State.TempRegisters[TempRegID] = + MRI.createGenericVirtualRegister(MatcherInfo.TypeObjects[TypeID]); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": TempRegs[" << TempRegID + << "] = GIR_MakeTempReg(" << TypeID << ")\n"); + break; + } + + case GIR_Coverage: { + int64_t RuleID = MatchTable[CurrentIdx++]; + CoverageInfo.setCovered(RuleID); + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); break; } case GIR_Done: - DEBUG(dbgs() << "GIR_Done"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_Done"); return true; default: diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h new file mode 100644 index 000000000000..e7945ff5bf4f --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -0,0 +1,287 @@ +//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h --===========// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This file contains some helper functions which try to cleanup artifacts +// such as G_TRUNCs/G_[ZSA]EXTENDS that were created during legalization to make +// the types match. This file also contains some combines of merges that happens +// at the end of the legalization. +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/GlobalISel/Legalizer.h" +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "legalizer" + +namespace llvm { +class LegalizationArtifactCombiner { + MachineIRBuilder &Builder; + MachineRegisterInfo &MRI; + const LegalizerInfo &LI; + +public: + LegalizationArtifactCombiner(MachineIRBuilder &B, MachineRegisterInfo &MRI, + const LegalizerInfo &LI) + : Builder(B), MRI(MRI), LI(LI) {} + + bool tryCombineAnyExt(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + if (MI.getOpcode() != TargetOpcode::G_ANYEXT) + return false; + if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, + MI.getOperand(1).getReg(), MRI)) { + DEBUG(dbgs() << ".. Combine MI: " << MI;); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = DefMI->getOperand(1).getReg(); + Builder.setInstr(MI); + // We get a copy/trunc/extend depending on the sizes + Builder.buildAnyExtOrTrunc(DstReg, SrcReg); + markInstAndDefDead(MI, *DefMI, DeadInsts); + return true; + } + return tryFoldImplicitDef(MI, DeadInsts); + } + + bool tryCombineZExt(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + + if (MI.getOpcode() != TargetOpcode::G_ZEXT) + return false; + if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, + MI.getOperand(1).getReg(), MRI)) { + unsigned DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (isInstUnsupported(TargetOpcode::G_AND, DstTy) || + isInstUnsupported(TargetOpcode::G_CONSTANT, DstTy)) + return false; + DEBUG(dbgs() << ".. Combine MI: " << MI;); + Builder.setInstr(MI); + unsigned ZExtSrc = MI.getOperand(1).getReg(); + LLT ZExtSrcTy = MRI.getType(ZExtSrc); + APInt Mask = APInt::getAllOnesValue(ZExtSrcTy.getSizeInBits()); + auto MaskCstMIB = Builder.buildConstant(DstTy, Mask.getZExtValue()); + unsigned TruncSrc = DefMI->getOperand(1).getReg(); + // We get a copy/trunc/extend depending on the sizes + auto SrcCopyOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc); + Builder.buildAnd(DstReg, SrcCopyOrTrunc, MaskCstMIB); + markInstAndDefDead(MI, *DefMI, DeadInsts); + return true; + } + return tryFoldImplicitDef(MI, DeadInsts); + } + + bool tryCombineSExt(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + + if (MI.getOpcode() != TargetOpcode::G_SEXT) + return false; + if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, + MI.getOperand(1).getReg(), MRI)) { + unsigned DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (isInstUnsupported(TargetOpcode::G_SHL, DstTy) || + isInstUnsupported(TargetOpcode::G_ASHR, DstTy) || + isInstUnsupported(TargetOpcode::G_CONSTANT, DstTy)) + return false; + DEBUG(dbgs() << ".. Combine MI: " << MI;); + Builder.setInstr(MI); + unsigned SExtSrc = MI.getOperand(1).getReg(); + LLT SExtSrcTy = MRI.getType(SExtSrc); + unsigned SizeDiff = DstTy.getSizeInBits() - SExtSrcTy.getSizeInBits(); + auto SizeDiffMIB = Builder.buildConstant(DstTy, SizeDiff); + unsigned TruncSrcReg = DefMI->getOperand(1).getReg(); + // We get a copy/trunc/extend depending on the sizes + auto SrcCopyExtOrTrunc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrcReg); + auto ShlMIB = Builder.buildInstr(TargetOpcode::G_SHL, DstTy, + SrcCopyExtOrTrunc, SizeDiffMIB); + Builder.buildInstr(TargetOpcode::G_ASHR, DstReg, ShlMIB, SizeDiffMIB); + markInstAndDefDead(MI, *DefMI, DeadInsts); + return true; + } + return tryFoldImplicitDef(MI, DeadInsts); + } + + /// Try to fold sb = EXTEND (G_IMPLICIT_DEF sa) -> sb = G_IMPLICIT_DEF + bool tryFoldImplicitDef(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + unsigned Opcode = MI.getOpcode(); + if (Opcode != TargetOpcode::G_ANYEXT && Opcode != TargetOpcode::G_ZEXT && + Opcode != TargetOpcode::G_SEXT) + return false; + + if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, + MI.getOperand(1).getReg(), MRI)) { + unsigned DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (isInstUnsupported(TargetOpcode::G_IMPLICIT_DEF, DstTy)) + return false; + DEBUG(dbgs() << ".. Combine EXT(IMPLICIT_DEF) " << MI;); + Builder.setInstr(MI); + Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, DstReg); + markInstAndDefDead(MI, *DefMI, DeadInsts); + return true; + } + return false; + } + + bool tryCombineMerges(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + + if (MI.getOpcode() != TargetOpcode::G_UNMERGE_VALUES) + return false; + + unsigned NumDefs = MI.getNumOperands() - 1; + unsigned SrcReg = MI.getOperand(NumDefs).getReg(); + MachineInstr *MergeI = MRI.getVRegDef(SrcReg); + if (!MergeI || (MergeI->getOpcode() != TargetOpcode::G_MERGE_VALUES)) + return false; + + const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; + + if (NumMergeRegs < NumDefs) { + if (NumDefs % NumMergeRegs != 0) + return false; + + Builder.setInstr(MI); + // Transform to UNMERGEs, for example + // %1 = G_MERGE_VALUES %4, %5 + // %9, %10, %11, %12 = G_UNMERGE_VALUES %1 + // to + // %9, %10 = G_UNMERGE_VALUES %4 + // %11, %12 = G_UNMERGE_VALUES %5 + + const unsigned NewNumDefs = NumDefs / NumMergeRegs; + for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) { + SmallVector<unsigned, 2> DstRegs; + for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs; + ++j, ++DefIdx) + DstRegs.push_back(MI.getOperand(DefIdx).getReg()); + + Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg()); + } + + } else if (NumMergeRegs > NumDefs) { + if (NumMergeRegs % NumDefs != 0) + return false; + + Builder.setInstr(MI); + // Transform to MERGEs + // %6 = G_MERGE_VALUES %17, %18, %19, %20 + // %7, %8 = G_UNMERGE_VALUES %6 + // to + // %7 = G_MERGE_VALUES %17, %18 + // %8 = G_MERGE_VALUES %19, %20 + + const unsigned NumRegs = NumMergeRegs / NumDefs; + for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) { + SmallVector<unsigned, 2> Regs; + for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs; + ++j, ++Idx) + Regs.push_back(MergeI->getOperand(Idx).getReg()); + + Builder.buildMerge(MI.getOperand(DefIdx).getReg(), Regs); + } + + } else { + // FIXME: is a COPY appropriate if the types mismatch? We know both + // registers are allocatable by now. + if (MRI.getType(MI.getOperand(0).getReg()) != + MRI.getType(MergeI->getOperand(1).getReg())) + return false; + + for (unsigned Idx = 0; Idx < NumDefs; ++Idx) + MRI.replaceRegWith(MI.getOperand(Idx).getReg(), + MergeI->getOperand(Idx + 1).getReg()); + } + + markInstAndDefDead(MI, *MergeI, DeadInsts); + return true; + } + + /// Try to combine away MI. + /// Returns true if it combined away the MI. + /// Adds instructions that are dead as a result of the combine + /// into DeadInsts, which can include MI. + bool tryCombineInstruction(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + switch (MI.getOpcode()) { + default: + return false; + case TargetOpcode::G_ANYEXT: + return tryCombineAnyExt(MI, DeadInsts); + case TargetOpcode::G_ZEXT: + return tryCombineZExt(MI, DeadInsts); + case TargetOpcode::G_SEXT: + return tryCombineSExt(MI, DeadInsts); + case TargetOpcode::G_UNMERGE_VALUES: + return tryCombineMerges(MI, DeadInsts); + case TargetOpcode::G_TRUNC: { + bool Changed = false; + for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg())) + Changed |= tryCombineInstruction(Use, DeadInsts); + return Changed; + } + } + } + +private: + /// Mark MI as dead. If a def of one of MI's operands, DefMI, would also be + /// dead due to MI being killed, then mark DefMI as dead too. + /// Some of the combines (extends(trunc)), try to walk through redundant + /// copies in between the extends and the truncs, and this attempts to collect + /// the in between copies if they're dead. + void markInstAndDefDead(MachineInstr &MI, MachineInstr &DefMI, + SmallVectorImpl<MachineInstr *> &DeadInsts) { + DeadInsts.push_back(&MI); + + // Collect all the copy instructions that are made dead, due to deleting + // this instruction. Collect all of them until the Trunc(DefMI). + // Eg, + // %1(s1) = G_TRUNC %0(s32) + // %2(s1) = COPY %1(s1) + // %3(s1) = COPY %2(s1) + // %4(s32) = G_ANYEXT %3(s1) + // In this case, we would have replaced %4 with a copy of %0, + // and as a result, %3, %2, %1 are dead. + MachineInstr *PrevMI = &MI; + while (PrevMI != &DefMI) { + // If we're dealing with G_UNMERGE_VALUES, tryCombineMerges doesn't really try + // to fold copies in between and we can ignore them here. + if (PrevMI->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) + break; + unsigned PrevRegSrc = PrevMI->getOperand(1).getReg(); + MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc); + if (MRI.hasOneUse(PrevRegSrc)) { + if (TmpDef != &DefMI) { + assert(TmpDef->getOpcode() == TargetOpcode::COPY && + "Expecting copy here"); + DeadInsts.push_back(TmpDef); + } + } else + break; + PrevMI = TmpDef; + } + if ((PrevMI == &DefMI || + DefMI.getOpcode() == TargetOpcode::G_MERGE_VALUES) && + MRI.hasOneUse(DefMI.getOperand(0).getReg())) + DeadInsts.push_back(&DefMI); + } + + /// Checks if the target legalizer info has specified anything about the + /// instruction, or if unsupported. + bool isInstUnsupported(unsigned Opcode, const LLT &DstTy) const { + auto Action = LI.getAction({Opcode, 0, DstTy}); + return Action.first == LegalizerInfo::LegalizeAction::Unsupported || + Action.first == LegalizerInfo::LegalizeAction::NotFound; + } +}; + +} // namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h index 9b9b8b563a30..8284ab6dac65 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -58,9 +58,6 @@ public: bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII); - bool combineMerges(MachineInstr &MI, MachineRegisterInfo &MRI, - const TargetInstrInfo &TII, MachineIRBuilder &MIRBuilder); - bool runOnMachineFunction(MachineFunction &MF) override; }; } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 1fd45b52e3ac..8bd8a9dcd0e2 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -89,6 +89,9 @@ public: /// functions MachineIRBuilder MIRBuilder; + /// Expose LegalizerInfo so the clients can re-use. + const LegalizerInfo &getLegalizerInfo() const { return LI; } + private: /// Helper function to split a wide generic register into bitwise blocks with diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index c259e93fdd36..b6735d538b37 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -20,11 +20,12 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/Support/LowLevelTypeImpl.h" -#include "llvm/Target/TargetOpcodes.h" -#include <cstdint> #include <cassert> +#include <cstdint> #include <tuple> +#include <unordered_map> #include <utility> namespace llvm { @@ -120,27 +121,144 @@ public: } } + typedef std::pair<uint16_t, LegalizeAction> SizeAndAction; + typedef std::vector<SizeAndAction> SizeAndActionsVec; + using SizeChangeStrategy = + std::function<SizeAndActionsVec(const SizeAndActionsVec &v)>; + /// More friendly way to set an action for common types that have an LLT /// representation. + /// The LegalizeAction must be one for which NeedsLegalizingToDifferentSize + /// returns false. void setAction(const InstrAspect &Aspect, LegalizeAction Action) { + assert(!needsLegalizingToDifferentSize(Action)); TablesInitialized = false; - unsigned Opcode = Aspect.Opcode - FirstOp; - if (Actions[Opcode].size() <= Aspect.Idx) - Actions[Opcode].resize(Aspect.Idx + 1); - Actions[Aspect.Opcode - FirstOp][Aspect.Idx][Aspect.Type] = Action; + const unsigned OpcodeIdx = Aspect.Opcode - FirstOp; + if (SpecifiedActions[OpcodeIdx].size() <= Aspect.Idx) + SpecifiedActions[OpcodeIdx].resize(Aspect.Idx + 1); + SpecifiedActions[OpcodeIdx][Aspect.Idx][Aspect.Type] = Action; } - /// If an operation on a given vector type (say <M x iN>) isn't explicitly - /// specified, we proceed in 2 stages. First we legalize the underlying scalar - /// (so that there's at least one legal vector with that scalar), then we - /// adjust the number of elements in the vector so that it is legal. The - /// desired action in the first step is controlled by this function. - void setScalarInVectorAction(unsigned Opcode, LLT ScalarTy, - LegalizeAction Action) { - assert(!ScalarTy.isVector()); - ScalarInVectorActions[std::make_pair(Opcode, ScalarTy)] = Action; + /// The setAction calls record the non-size-changing legalization actions + /// to take on specificly-sized types. The SizeChangeStrategy defines what + /// to do when the size of the type needs to be changed to reach a legally + /// sized type (i.e., one that was defined through a setAction call). + /// e.g. + /// setAction ({G_ADD, 0, LLT::scalar(32)}, Legal); + /// setLegalizeScalarToDifferentSizeStrategy( + /// G_ADD, 0, widenToLargerTypesAndNarrowToLargest); + /// will end up defining getAction({G_ADD, 0, T}) to return the following + /// actions for different scalar types T: + /// LLT::scalar(1)..LLT::scalar(31): {WidenScalar, 0, LLT::scalar(32)} + /// LLT::scalar(32): {Legal, 0, LLT::scalar(32)} + /// LLT::scalar(33)..: {NarrowScalar, 0, LLT::scalar(32)} + /// + /// If no SizeChangeAction gets defined, through this function, + /// the default is unsupportedForDifferentSizes. + void setLegalizeScalarToDifferentSizeStrategy(const unsigned Opcode, + const unsigned TypeIdx, + SizeChangeStrategy S) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (ScalarSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) + ScalarSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); + ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; + } + + /// See also setLegalizeScalarToDifferentSizeStrategy. + /// This function allows to set the SizeChangeStrategy for vector elements. + void setLegalizeVectorElementToDifferentSizeStrategy(const unsigned Opcode, + const unsigned TypeIdx, + SizeChangeStrategy S) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (VectorElementSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) + VectorElementSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); + VectorElementSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; + } + + /// A SizeChangeStrategy for the common case where legalization for a + /// particular operation consists of only supporting a specific set of type + /// sizes. E.g. + /// setAction ({G_DIV, 0, LLT::scalar(32)}, Legal); + /// setAction ({G_DIV, 0, LLT::scalar(64)}, Legal); + /// setLegalizeScalarToDifferentSizeStrategy( + /// G_DIV, 0, unsupportedForDifferentSizes); + /// will result in getAction({G_DIV, 0, T}) to return Legal for s32 and s64, + /// and Unsupported for all other scalar types T. + static SizeAndActionsVec + unsupportedForDifferentSizes(const SizeAndActionsVec &v) { + return increaseToLargerTypesAndDecreaseToLargest(v, Unsupported, + Unsupported); } + /// A SizeChangeStrategy for the common case where legalization for a + /// particular operation consists of widening the type to a large legal type, + /// unless there is no such type and then instead it should be narrowed to the + /// largest legal type. + static SizeAndActionsVec + widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) { + assert(v.size() > 0 && + "At least one size that can be legalized towards is needed" + " for this SizeChangeStrategy"); + return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, + NarrowScalar); + } + + static SizeAndActionsVec + widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec &v) { + return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, + Unsupported); + } + + static SizeAndActionsVec + narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec &v) { + return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, + Unsupported); + } + + static SizeAndActionsVec + narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) { + assert(v.size() > 0 && + "At least one size that can be legalized towards is needed" + " for this SizeChangeStrategy"); + return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, + WidenScalar); + } + + /// A SizeChangeStrategy for the common case where legalization for a + /// particular vector operation consists of having more elements in the + /// vector, to a type that is legal. Unless there is no such type and then + /// instead it should be legalized towards the widest vector that's still + /// legal. E.g. + /// setAction({G_ADD, LLT::vector(8, 8)}, Legal); + /// setAction({G_ADD, LLT::vector(16, 8)}, Legal); + /// setAction({G_ADD, LLT::vector(2, 32)}, Legal); + /// setAction({G_ADD, LLT::vector(4, 32)}, Legal); + /// setLegalizeVectorElementToDifferentSizeStrategy( + /// G_ADD, 0, moreToWiderTypesAndLessToWidest); + /// will result in the following getAction results: + /// * getAction({G_ADD, LLT::vector(8,8)}) returns + /// (Legal, vector(8,8)). + /// * getAction({G_ADD, LLT::vector(9,8)}) returns + /// (MoreElements, vector(16,8)). + /// * getAction({G_ADD, LLT::vector(8,32)}) returns + /// (FewerElements, vector(4,32)). + static SizeAndActionsVec + moreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) { + return increaseToLargerTypesAndDecreaseToLargest(v, MoreElements, + FewerElements); + } + + /// Helper function to implement many typical SizeChangeStrategy functions. + static SizeAndActionsVec + increaseToLargerTypesAndDecreaseToLargest(const SizeAndActionsVec &v, + LegalizeAction IncreaseAction, + LegalizeAction DecreaseAction); + /// Helper function to implement many typical SizeChangeStrategy functions. + static SizeAndActionsVec + decreaseToSmallerTypesAndIncreaseToSmallest(const SizeAndActionsVec &v, + LegalizeAction DecreaseAction, + LegalizeAction IncreaseAction); + /// Determine what action should be taken to legalize the given generic /// instruction opcode, type-index and type. Requires computeTables to have /// been called. @@ -158,55 +276,6 @@ public: std::tuple<LegalizeAction, unsigned, LLT> getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; - /// Iterate the given function (typically something like doubling the width) - /// on Ty until we find a legal type for this operation. - Optional<LLT> findLegalizableSize(const InstrAspect &Aspect, - function_ref<LLT(LLT)> NextType) const { - LegalizeAction Action; - const TypeMap &Map = Actions[Aspect.Opcode - FirstOp][Aspect.Idx]; - LLT Ty = Aspect.Type; - do { - Ty = NextType(Ty); - auto ActionIt = Map.find(Ty); - if (ActionIt == Map.end()) { - auto DefaultIt = DefaultActions.find(Aspect.Opcode); - if (DefaultIt == DefaultActions.end()) - return None; - Action = DefaultIt->second; - } else - Action = ActionIt->second; - } while (needsLegalizingToDifferentSize(Action)); - return Ty; - } - - /// Find what type it's actually OK to perform the given operation on, given - /// the general approach we've decided to take. - Optional<LLT> findLegalType(const InstrAspect &Aspect, LegalizeAction Action) const; - - std::pair<LegalizeAction, LLT> findLegalAction(const InstrAspect &Aspect, - LegalizeAction Action) const { - auto LegalType = findLegalType(Aspect, Action); - if (!LegalType) - return std::make_pair(LegalizeAction::Unsupported, LLT()); - return std::make_pair(Action, *LegalType); - } - - /// Find the specified \p Aspect in the primary (explicitly set) Actions - /// table. Returns either the action the target requested or NotFound if there - /// was no setAction call. - LegalizeAction findInActions(const InstrAspect &Aspect) const { - if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp) - return NotFound; - if (Aspect.Idx >= Actions[Aspect.Opcode - FirstOp].size()) - return NotFound; - const TypeMap &Map = Actions[Aspect.Opcode - FirstOp][Aspect.Idx]; - auto ActionIt = Map.find(Aspect.Type); - if (ActionIt == Map.end()) - return NotFound; - - return ActionIt->second; - } - bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; virtual bool legalizeCustom(MachineInstr &MI, @@ -214,20 +283,181 @@ public: MachineIRBuilder &MIRBuilder) const; private: - static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; - static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; + /// The SizeAndActionsVec is a representation mapping between all natural + /// numbers and an Action. The natural number represents the bit size of + /// the InstrAspect. For example, for a target with native support for 32-bit + /// and 64-bit additions, you'd express that as: + /// setScalarAction(G_ADD, 0, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Legal}, // bit sizes [32, 33[ + /// {33, WidenScalar}, // bit sizes [33, 64[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, NarrowScalar} // bit sizes [65, +inf[ + /// }); + /// It may be that only 64-bit pointers are supported on your target: + /// setPointerAction(G_GEP, 0, LLT:pointer(1), + /// {{1, Unsupported}, // bit sizes [ 1, 63[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, Unsupported}, // bit sizes [65, +inf[ + /// }); + void setScalarAction(const unsigned Opcode, const unsigned TypeIndex, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + SmallVector<SizeAndActionsVec, 1> &Actions = ScalarActions[OpcodeIdx]; + setActions(TypeIndex, Actions, SizeAndActions); + } + void setPointerAction(const unsigned Opcode, const unsigned TypeIndex, + const unsigned AddressSpace, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace) == + AddrSpace2PointerActions[OpcodeIdx].end()) + AddrSpace2PointerActions[OpcodeIdx][AddressSpace] = {{}}; + SmallVector<SizeAndActionsVec, 1> &Actions = + AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace)->second; + setActions(TypeIndex, Actions, SizeAndActions); + } - using TypeMap = DenseMap<LLT, LegalizeAction>; - using SIVActionMap = DenseMap<std::pair<unsigned, LLT>, LegalizeAction>; + /// If an operation on a given vector type (say <M x iN>) isn't explicitly + /// specified, we proceed in 2 stages. First we legalize the underlying scalar + /// (so that there's at least one legal vector with that scalar), then we + /// adjust the number of elements in the vector so that it is legal. The + /// desired action in the first step is controlled by this function. + void setScalarInVectorAction(const unsigned Opcode, const unsigned TypeIndex, + const SizeAndActionsVec &SizeAndActions) { + unsigned OpcodeIdx = Opcode - FirstOp; + SmallVector<SizeAndActionsVec, 1> &Actions = + ScalarInVectorActions[OpcodeIdx]; + setActions(TypeIndex, Actions, SizeAndActions); + } + + /// See also setScalarInVectorAction. + /// This function let's you specify the number of elements in a vector that + /// are legal for a legal element size. + void setVectorNumElementAction(const unsigned Opcode, + const unsigned TypeIndex, + const unsigned ElementSize, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (NumElements2Actions[OpcodeIdx].find(ElementSize) == + NumElements2Actions[OpcodeIdx].end()) + NumElements2Actions[OpcodeIdx][ElementSize] = {{}}; + SmallVector<SizeAndActionsVec, 1> &Actions = + NumElements2Actions[OpcodeIdx].find(ElementSize)->second; + setActions(TypeIndex, Actions, SizeAndActions); + } - SmallVector<TypeMap, 1> Actions[LastOp - FirstOp + 1]; - SIVActionMap ScalarInVectorActions; - DenseMap<std::pair<unsigned, LLT>, uint16_t> MaxLegalVectorElts; - DenseMap<unsigned, LegalizeAction> DefaultActions; + /// A partial SizeAndActionsVec potentially doesn't cover all bit sizes, + /// i.e. it's OK if it doesn't start from size 1. + static void checkPartialSizeAndActionsVector(const SizeAndActionsVec& v) { +#ifndef NDEBUG + // The sizes should be in increasing order + int prev_size = -1; + for(auto SizeAndAction: v) { + assert(SizeAndAction.first > prev_size); + prev_size = SizeAndAction.first; + } + // - for every Widen action, there should be a larger bitsize that + // can be legalized towards (e.g. Legal, Lower, Libcall or Custom + // action). + // - for every Narrow action, there should be a smaller bitsize that + // can be legalized towards. + int SmallestNarrowIdx = -1; + int LargestWidenIdx = -1; + int SmallestLegalizableToSameSizeIdx = -1; + int LargestLegalizableToSameSizeIdx = -1; + for(size_t i=0; i<v.size(); ++i) { + switch (v[i].second) { + case FewerElements: + case NarrowScalar: + if (SmallestNarrowIdx == -1) + SmallestNarrowIdx = i; + break; + case WidenScalar: + case MoreElements: + LargestWidenIdx = i; + break; + case Unsupported: + break; + default: + if (SmallestLegalizableToSameSizeIdx == -1) + SmallestLegalizableToSameSizeIdx = i; + LargestLegalizableToSameSizeIdx = i; + } + } + if (SmallestNarrowIdx != -1) { + assert(SmallestLegalizableToSameSizeIdx != -1); + assert(SmallestNarrowIdx > SmallestLegalizableToSameSizeIdx); + } + if (LargestWidenIdx != -1) + assert(LargestWidenIdx < LargestLegalizableToSameSizeIdx); +#endif + } + + /// A full SizeAndActionsVec must cover all bit sizes, i.e. must start with + /// from size 1. + static void checkFullSizeAndActionsVector(const SizeAndActionsVec& v) { +#ifndef NDEBUG + // Data structure invariant: The first bit size must be size 1. + assert(v.size() >= 1); + assert(v[0].first == 1); + checkPartialSizeAndActionsVector(v); +#endif + } + + /// Sets actions for all bit sizes on a particular generic opcode, type + /// index and scalar or pointer type. + void setActions(unsigned TypeIndex, + SmallVector<SizeAndActionsVec, 1> &Actions, + const SizeAndActionsVec &SizeAndActions) { + checkFullSizeAndActionsVector(SizeAndActions); + if (Actions.size() <= TypeIndex) + Actions.resize(TypeIndex + 1); + Actions[TypeIndex] = SizeAndActions; + } + + static SizeAndAction findAction(const SizeAndActionsVec &Vec, + const uint32_t Size); + + /// Returns the next action needed to get the scalar or pointer type closer + /// to being legal + /// E.g. findLegalAction({G_REM, 13}) should return + /// (WidenScalar, 32). After that, findLegalAction({G_REM, 32}) will + /// probably be called, which should return (Lower, 32). + /// This is assuming the setScalarAction on G_REM was something like: + /// setScalarAction(G_REM, 0, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Lower}, // bit sizes [32, 33[ + /// {33, NarrowScalar} // bit sizes [65, +inf[ + /// }); + std::pair<LegalizeAction, LLT> + findScalarLegalAction(const InstrAspect &Aspect) const; + + /// Returns the next action needed towards legalizing the vector type. + std::pair<LegalizeAction, LLT> + findVectorLegalAction(const InstrAspect &Aspect) const; + + static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; + static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; - bool TablesInitialized = false; + // Data structures used temporarily during construction of legality data: + typedef DenseMap<LLT, LegalizeAction> TypeMap; + SmallVector<TypeMap, 1> SpecifiedActions[LastOp - FirstOp + 1]; + SmallVector<SizeChangeStrategy, 1> + ScalarSizeChangeStrategies[LastOp - FirstOp + 1]; + SmallVector<SizeChangeStrategy, 1> + VectorElementSizeChangeStrategies[LastOp - FirstOp + 1]; + bool TablesInitialized; + + // Data structures used by getAction: + SmallVector<SizeAndActionsVec, 1> ScalarActions[LastOp - FirstOp + 1]; + SmallVector<SizeAndActionsVec, 1> ScalarInVectorActions[LastOp - FirstOp + 1]; + std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> + AddrSpace2PointerActions[LastOp - FirstOp + 1]; + std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> + NumElements2Actions[LastOp - FirstOp + 1]; }; -} // end namespace llvm +} // end namespace llvm. #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 85e6fef1f3c2..aa875c11d86f 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -70,13 +70,33 @@ class MachineIRBuilder { return getMF().getRegInfo().createVirtualRegister(RC); } - unsigned getRegFromArg(unsigned Reg) { return Reg; } + void addUseFromArg(MachineInstrBuilder &MIB, unsigned Reg) { + MIB.addUse(Reg); + } + void addUseFromArg(MachineInstrBuilder &MIB, const MachineInstrBuilder &UseMIB) { + MIB.addUse(UseMIB->getOperand(0).getReg()); + } + + void addUsesFromArgs(MachineInstrBuilder &MIB) { } + template<typename UseArgTy, typename ... UseArgsTy> + void addUsesFromArgs(MachineInstrBuilder &MIB, UseArgTy &&Arg1, UseArgsTy &&... Args) { + addUseFromArg(MIB, Arg1); + addUsesFromArgs(MIB, std::forward<UseArgsTy>(Args)...); + } + unsigned getRegFromArg(unsigned Reg) { return Reg; } unsigned getRegFromArg(const MachineInstrBuilder &MIB) { return MIB->getOperand(0).getReg(); } public: + /// Some constructors for easy use. + MachineIRBuilder() = default; + MachineIRBuilder(MachineFunction &MF) { setMF(MF); } + MachineIRBuilder(MachineInstr &MI) : MachineIRBuilder(*MI.getMF()) { + setInstr(MI); + } + /// Getter for the function we currently build. MachineFunction &getMF() { assert(MF && "MachineFunction is not set"); @@ -146,9 +166,7 @@ public: MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, UseArgsTy &&... Args) { auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - unsigned It[] = {(getRegFromArg(Args))...}; - for (const auto &i : It) - MIB.addUse(i); + addUsesFromArgs(MIB, std::forward<UseArgsTy>(Args)...); return MIB; } @@ -168,11 +186,12 @@ public: const MDNode *Expr); /// Build and insert a DBG_VALUE instruction expressing the fact that the - /// associated \p Variable lives in memory at \p Reg + \p Offset (suitably - /// modified by \p Expr). - MachineInstrBuilder buildIndirectDbgValue(unsigned Reg, unsigned Offset, + /// associated \p Variable lives in memory at \p Reg (suitably modified by \p + /// Expr). + MachineInstrBuilder buildIndirectDbgValue(unsigned Reg, const MDNode *Variable, const MDNode *Expr); + /// Build and insert a DBG_VALUE instruction expressing the fact that the /// associated \p Variable lives in the stack slot specified by \p FI /// (suitably modified by \p Expr). @@ -181,11 +200,11 @@ public: /// Build and insert a DBG_VALUE instructions specifying that \p Variable is /// given by \p C (suitably modified by \p Expr). - MachineInstrBuilder buildConstDbgValue(const Constant &C, unsigned Offset, + MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr); - /// Build and insert \p Res<def> = G_FRAME_INDEX \p Idx + /// Build and insert \p Res = G_FRAME_INDEX \p Idx /// /// G_FRAME_INDEX materializes the address of an alloca value or other /// stack-based object. @@ -196,7 +215,7 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildFrameIndex(unsigned Res, int Idx); - /// Build and insert \p Res<def> = G_GLOBAL_VALUE \p GV + /// Build and insert \p Res = G_GLOBAL_VALUE \p GV /// /// G_GLOBAL_VALUE materializes the address of the specified global /// into \p Res. @@ -208,7 +227,7 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildGlobalValue(unsigned Res, const GlobalValue *GV); - /// Build and insert \p Res<def> = G_ADD \p Op0, \p Op1 + /// Build and insert \p Res = G_ADD \p Op0, \p Op1 /// /// G_ADD sets \p Res to the sum of integer parameters \p Op0 and \p Op1, /// truncated to their width. @@ -226,7 +245,7 @@ public: return buildAdd(Res, (getRegFromArg(UseArgs))...); } - /// Build and insert \p Res<def> = G_SUB \p Op0, \p Op1 + /// Build and insert \p Res = G_SUB \p Op0, \p Op1 /// /// G_SUB sets \p Res to the sum of integer parameters \p Op0 and \p Op1, /// truncated to their width. @@ -239,7 +258,7 @@ public: MachineInstrBuilder buildSub(unsigned Res, unsigned Op0, unsigned Op1); - /// Build and insert \p Res<def> = G_MUL \p Op0, \p Op1 + /// Build and insert \p Res = G_MUL \p Op0, \p Op1 /// /// G_MUL sets \p Res to the sum of integer parameters \p Op0 and \p Op1, /// truncated to their width. @@ -252,7 +271,7 @@ public: MachineInstrBuilder buildMul(unsigned Res, unsigned Op0, unsigned Op1); - /// Build and insert \p Res<def> = G_GEP \p Op0, \p Op1 + /// Build and insert \p Res = G_GEP \p Op0, \p Op1 /// /// G_GEP adds \p Op1 bytes to the pointer specified by \p Op0, /// storing the resulting pointer in \p Res. @@ -266,7 +285,7 @@ public: MachineInstrBuilder buildGEP(unsigned Res, unsigned Op0, unsigned Op1); - /// Materialize and insert \p Res<def> = G_GEP \p Op0, (G_CONSTANT \p Value) + /// Materialize and insert \p Res = G_GEP \p Op0, (G_CONSTANT \p Value) /// /// G_GEP adds \p Value bytes to the pointer specified by \p Op0, /// storing the resulting pointer in \p Res. If \p Value is zero then no @@ -286,7 +305,7 @@ public: const LLT &ValueTy, uint64_t Value); - /// Build and insert \p Res<def> = G_PTR_MASK \p Op0, \p NumBits + /// Build and insert \p Res = G_PTR_MASK \p Op0, \p NumBits /// /// G_PTR_MASK clears the low bits of a pointer operand without destroying its /// pointer properties. This has the effect of rounding the address *down* to @@ -302,7 +321,7 @@ public: MachineInstrBuilder buildPtrMask(unsigned Res, unsigned Op0, uint32_t NumBits); - /// Build and insert \p Res<def>, \p CarryOut<def> = G_UADDE \p Op0, + /// Build and insert \p Res, \p CarryOut = G_UADDE \p Op0, /// \p Op1, \p CarryIn /// /// G_UADDE sets \p Res to \p Op0 + \p Op1 + \p CarryIn (truncated to the bit @@ -319,7 +338,7 @@ public: MachineInstrBuilder buildUAdde(unsigned Res, unsigned CarryOut, unsigned Op0, unsigned Op1, unsigned CarryIn); - /// Build and insert \p Res<def> = G_AND \p Op0, \p Op1 + /// Build and insert \p Res = G_AND \p Op0, \p Op1 /// /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p /// Op1. @@ -329,10 +348,14 @@ public: /// with the same (scalar or vector) type). /// /// \return a MachineInstrBuilder for the newly created instruction. + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildAnd(DstTy &&Dst, UseArgsTy &&... UseArgs) { + return buildAnd(getDestFromArg(Dst), getRegFromArg(UseArgs)...); + } MachineInstrBuilder buildAnd(unsigned Res, unsigned Op0, unsigned Op1); - /// Build and insert \p Res<def> = G_OR \p Op0, \p Op1 + /// Build and insert \p Res = G_OR \p Op0, \p Op1 /// /// G_OR sets \p Res to the bitwise or of integer parameters \p Op0 and \p /// Op1. @@ -344,7 +367,7 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildOr(unsigned Res, unsigned Op0, unsigned Op1); - /// Build and insert \p Res<def> = G_ANYEXT \p Op0 + /// Build and insert \p Res = G_ANYEXT \p Op0 /// /// G_ANYEXT produces a register of the specified width, with bits 0 to /// sizeof(\p Ty) * 8 set to \p Op. The remaining bits are unspecified @@ -357,9 +380,14 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. + MachineInstrBuilder buildAnyExt(unsigned Res, unsigned Op); + template <typename DstType, typename ArgType> + MachineInstrBuilder buildAnyExt(DstType &&Res, ArgType &&Arg) { + return buildAnyExt(getDestFromArg(Res), getRegFromArg(Arg)); + } - /// Build and insert \p Res<def> = G_SEXT \p Op + /// Build and insert \p Res = G_SEXT \p Op /// /// G_SEXT produces a register of the specified width, with bits 0 to /// sizeof(\p Ty) * 8 set to \p Op. The remaining bits are duplicated from the @@ -373,7 +401,7 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildSExt(unsigned Res, unsigned Op); - /// Build and insert \p Res<def> = G_ZEXT \p Op + /// Build and insert \p Res = G_ZEXT \p Op /// /// G_ZEXT produces a register of the specified width, with bits 0 to /// sizeof(\p Ty) * 8 set to \p Op. The remaining bits are 0. For a vector @@ -387,7 +415,7 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildZExt(unsigned Res, unsigned Op); - /// Build and insert \p Res<def> = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or + /// Build and insert \p Res = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. /// /// /// \pre setBasicBlock or setMI must have been called. @@ -397,7 +425,7 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildSExtOrTrunc(unsigned Res, unsigned Op); - /// Build and insert \p Res<def> = G_ZEXT \p Op, \p Res = G_TRUNC \p Op, or + /// Build and insert \p Res = G_ZEXT \p Op, \p Res = G_TRUNC \p Op, or /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. /// /// /// \pre setBasicBlock or setMI must have been called. @@ -407,6 +435,32 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildZExtOrTrunc(unsigned Res, unsigned Op); + // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_TRUNC \p Op, or + /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op. + /// /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with scalar or vector type. + /// \pre \p Op must be a generic virtual register with scalar or vector type. + /// + /// \return The newly created instruction. + template <typename DstTy, typename UseArgTy> + MachineInstrBuilder buildAnyExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { + return buildAnyExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); + } + MachineInstrBuilder buildAnyExtOrTrunc(unsigned Res, unsigned Op); + + /// Build and insert \p Res = \p ExtOpc, \p Res = G_TRUNC \p + /// Op, or \p Res = COPY \p Op depending on the differing sizes of \p Res and + /// \p Op. + /// /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with scalar or vector type. + /// \pre \p Op must be a generic virtual register with scalar or vector type. + /// + /// \return The newly created instruction. + MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, unsigned Res, + unsigned Op); + /// Build and insert an appropriate cast between two registers of equal size. MachineInstrBuilder buildCast(unsigned Dst, unsigned Src); @@ -480,7 +534,7 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildFConstant(unsigned Res, const ConstantFP &Val); - /// Build and insert \p Res<def> = COPY Op + /// Build and insert \p Res = COPY Op /// /// Register-to-register COPY sets \p Res to \p Op. /// @@ -488,8 +542,12 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildCopy(unsigned Res, unsigned Op); + template <typename DstType, typename SrcType> + MachineInstrBuilder buildCopy(DstType &&Res, SrcType &&Src) { + return buildCopy(getDestFromArg(Res), getRegFromArg(Src)); + } - /// Build and insert `Res<def> = G_LOAD Addr, MMO`. + /// Build and insert `Res = G_LOAD Addr, MMO`. /// /// Loads the value stored at \p Addr. Puts the result in \p Res. /// @@ -513,7 +571,7 @@ public: MachineInstrBuilder buildStore(unsigned Val, unsigned Addr, MachineMemOperand &MMO); - /// Build and insert `Res0<def>, ... = G_EXTRACT Src, Idx0`. + /// Build and insert `Res0, ... = G_EXTRACT Src, Idx0`. /// /// \pre setBasicBlock or setMI must have been called. /// \pre \p Res and \p Src must be generic virtual registers. @@ -540,7 +598,7 @@ public: void buildSequence(unsigned Res, ArrayRef<unsigned> Ops, ArrayRef<uint64_t> Indices); - /// Build and insert \p Res<def> = G_MERGE_VALUES \p Op0, ... + /// Build and insert \p Res = G_MERGE_VALUES \p Op0, ... /// /// G_MERGE_VALUES combines the input elements contiguously into a larger /// register. @@ -553,7 +611,7 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildMerge(unsigned Res, ArrayRef<unsigned> Ops); - /// Build and insert \p Res0<def>, ... = G_UNMERGE_VALUES \p Op + /// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op /// /// G_UNMERGE_VALUES splits contiguous bits of the input into multiple /// @@ -581,7 +639,7 @@ public: MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, unsigned Res, bool HasSideEffects); - /// Build and insert \p Res<def> = G_FPTRUNC \p Op + /// Build and insert \p Res = G_FPTRUNC \p Op /// /// G_FPTRUNC converts a floating-point value into one with a smaller type. /// @@ -593,7 +651,7 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildFPTrunc(unsigned Res, unsigned Op); - /// Build and insert \p Res<def> = G_TRUNC \p Op + /// Build and insert \p Res = G_TRUNC \p Op /// /// G_TRUNC extracts the low bits of a type. For a vector type each element is /// truncated independently before being packed into the destination. @@ -605,6 +663,10 @@ public: /// /// \return The newly created instruction. MachineInstrBuilder buildTrunc(unsigned Res, unsigned Op); + template <typename DstType, typename SrcType> + MachineInstrBuilder buildTrunc(DstType &&Res, SrcType &&Src) { + return buildTrunc(getDestFromArg(Res), getRegFromArg(Src)); + } /// Build and insert a \p Res = G_ICMP \p Pred, \p Op0, \p Op1 /// @@ -649,7 +711,7 @@ public: MachineInstrBuilder buildSelect(unsigned Res, unsigned Tst, unsigned Op0, unsigned Op1); - /// Build and insert \p Res<def> = G_INSERT_VECTOR_ELT \p Val, + /// Build and insert \p Res = G_INSERT_VECTOR_ELT \p Val, /// \p Elt, \p Idx /// /// \pre setBasicBlock or setMI must have been called. @@ -662,7 +724,7 @@ public: MachineInstrBuilder buildInsertVectorElement(unsigned Res, unsigned Val, unsigned Elt, unsigned Idx); - /// Build and insert \p Res<def> = G_EXTRACT_VECTOR_ELT \p Val, \p Idx + /// Build and insert \p Res = G_EXTRACT_VECTOR_ELT \p Val, \p Idx /// /// \pre setBasicBlock or setMI must have been called. /// \pre \p Res must be a generic virtual register with scalar type. @@ -672,6 +734,24 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val, unsigned Idx); + + /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, + /// MMO`. + /// + /// Atomically replace the value at \p Addr with \p NewVal if it is currently + /// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p + /// Addr in \p Res. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register of scalar type. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual + /// registers of the same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, + unsigned CmpVal, unsigned NewVal, + MachineMemOperand &MMO); }; } // End namespace llvm. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index 60905c7ec226..02868b220984 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -407,6 +407,10 @@ protected: mutable DenseMap<unsigned, std::unique_ptr<const InstructionMapping>> MapOfInstructionMappings; + /// Getting the minimal register class of a physreg is expensive. + /// Cache this information as we get it. + mutable DenseMap<unsigned, const TargetRegisterClass *> PhysRegMinimalRCs; + /// Create a RegisterBankInfo that can accommodate up to \p NumRegBanks /// RegisterBank instances. RegisterBankInfo(RegisterBank **RegBanks, unsigned NumRegBanks); @@ -427,6 +431,11 @@ protected: return *RegBanks[ID]; } + /// Get the MinimalPhysRegClass for Reg. + /// \pre Reg is a physical register. + const TargetRegisterClass & + getMinimalPhysRegClass(unsigned Reg, const TargetRegisterInfo &TRI) const; + /// Try to get the mapping of \p MI. /// See getInstrMapping for more details on what a mapping represents. /// @@ -699,8 +708,8 @@ public: /// virtual register. /// /// \pre \p Reg != 0 (NoRegister). - static unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI); + unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; /// Check that information hold by this instance make sense for the /// given \p TRI. diff --git a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 50ddbeb9432a..5864c15cc8eb 100644 --- a/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/contrib/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -79,5 +79,11 @@ Optional<int64_t> getConstantVRegVal(unsigned VReg, const ConstantFP* getConstantFPVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); +/// See if Reg is defined by an single def instruction that is +/// Opcode. Also try to do trivial folding if it's a COPY with +/// same types. Returns null otherwise. +MachineInstr *getOpcodeDef(unsigned Opcode, unsigned Reg, + const MachineRegisterInfo &MRI); + } // End namespace llvm. #endif diff --git a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h index bc5d2353f63e..d256849be9af 100644 --- a/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/contrib/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -186,7 +186,8 @@ namespace ISD { /// BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways. /// Given two values of the same integer value type, this produces a value /// twice as big. Like EXTRACT_ELEMENT, this can only be used before - /// legalization. + /// legalization. The lower part of the composite value should be in + /// element 0 and the upper part should be in element 1. BUILD_PAIR, /// MERGE_VALUES - This node takes multiple discrete operands and returns @@ -263,6 +264,7 @@ namespace ISD { /// They are used to limit optimizations while the DAG is being /// optimized. STRICT_FADD, STRICT_FSUB, STRICT_FMUL, STRICT_FDIV, STRICT_FREM, + STRICT_FMA, /// Constrained versions of libm-equivalent floating point intrinsics. /// These will be lowered to the equivalent non-constrained pseudo-op @@ -637,6 +639,12 @@ namespace ISD { /// take a chain as input and return a chain. EH_LABEL, + /// ANNOTATION_LABEL - Represents a mid basic block label used by + /// annotations. This should remain within the basic block and be ordered + /// with respect to other call instructions, but loads and stores may float + /// past it. + ANNOTATION_LABEL, + /// CATCHPAD - Represents a catchpad instruction. CATCHPAD, @@ -831,7 +839,7 @@ namespace ISD { /// which do not reference a specific memory location should be less than /// this value. Those that do must not be less than this value, and can /// be used with SelectionDAG::getMemIntrinsicNode. - static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+300; + static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+400; //===--------------------------------------------------------------------===// /// MemIndexedMode enum - This enum defines the load / store indexed diff --git a/contrib/llvm/include/llvm/CodeGen/IntrinsicLowering.h b/contrib/llvm/include/llvm/CodeGen/IntrinsicLowering.h index a404b9b70d3a..597d684909c1 100644 --- a/contrib/llvm/include/llvm/CodeGen/IntrinsicLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/IntrinsicLowering.h @@ -31,26 +31,22 @@ class IntrinsicLowering { public: explicit IntrinsicLowering(const DataLayout &DL) : DL(DL), Warned(false) {} - /// AddPrototypes - This method, if called, causes all of the prototypes - /// that might be needed by an intrinsic lowering implementation to be - /// inserted into the module specified. + /// Add all of the prototypes that might be needed by an intrinsic lowering + /// implementation to be inserted into the module specified. void AddPrototypes(Module &M); - /// LowerIntrinsicCall - This method replaces a call with the LLVM function - /// which should be used to implement the specified intrinsic function call. + /// Replace a call to the specified intrinsic function. /// If an intrinsic function must be implemented by the code generator /// (such as va_start), this function should print a message and abort. /// /// Otherwise, if an intrinsic function call can be lowered, the code to /// implement it (often a call to a non-intrinsic function) is inserted - /// _after_ the call instruction and the call is deleted. The caller must + /// _after_ the call instruction and the call is deleted. The caller must /// be capable of handling this kind of change. - /// void LowerIntrinsicCall(CallInst *CI); - /// LowerToByteSwap - Replace a call instruction into a call to bswap - /// intrinsic. Return false if it has determined the call is not a - /// simple integer bswap. + /// Try to replace a call instruction with a call to a bswap intrinsic. Return + /// false if the call is not a simple integer bswap. static bool LowerToByteSwap(CallInst *CI); }; } diff --git a/contrib/llvm/include/llvm/CodeGen/LatencyPriorityQueue.h b/contrib/llvm/include/llvm/CodeGen/LatencyPriorityQueue.h index f347f66e0981..988e6d6cb3a3 100644 --- a/contrib/llvm/include/llvm/CodeGen/LatencyPriorityQueue.h +++ b/contrib/llvm/include/llvm/CodeGen/LatencyPriorityQueue.h @@ -22,7 +22,7 @@ namespace llvm { class LatencyPriorityQueue; /// Sorting functions for the Available queue. - struct latency_sort : public std::binary_function<SUnit*, SUnit*, bool> { + struct latency_sort { LatencyPriorityQueue *PQ; explicit latency_sort(LatencyPriorityQueue *pq) : PQ(pq) {} diff --git a/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h b/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h index 820e88362483..1150f3c1c47b 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveIntervals.h @@ -1,4 +1,4 @@ -//===- LiveIntervalAnalysis.h - Live Interval Analysis ----------*- C++ -*-===// +//===- LiveIntervals.h - Live Interval Analysis -----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -17,8 +17,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_LIVEINTERVALANALYSIS_H -#define LLVM_CODEGEN_LIVEINTERVALANALYSIS_H +#ifndef LLVM_CODEGEN_LIVEINTERVALS_H +#define LLVM_CODEGEN_LIVEINTERVALS_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/IndexedMap.h" @@ -28,11 +28,11 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/LaneBitmask.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <cstdint> #include <utility> @@ -107,6 +107,11 @@ class VirtRegMap; const MachineBlockFrequencyInfo *MBFI, const MachineInstr &Instr); + /// Calculate the spill weight to assign to a single instruction. + static float getSpillWeight(bool isDef, bool isUse, + const MachineBlockFrequencyInfo *MBFI, + const MachineBasicBlock *MBB); + LiveInterval &getInterval(unsigned Reg) { if (hasInterval(Reg)) return *VirtRegIntervals[Reg]; @@ -473,4 +478,4 @@ class VirtRegMap; } // end namespace llvm -#endif // LLVM_CODEGEN_LIVEINTERVALANALYSIS_H +#endif diff --git a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h index f9c741dd75b2..f9aab0d09e1f 100644 --- a/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/contrib/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -20,11 +20,11 @@ /// register. /// /// X86 Example: -/// %YMM0<def> = ... -/// %XMM0<def> = ... (Kills %XMM0, all %XMM0s sub-registers, and %YMM0) +/// %ymm0 = ... +/// %xmm0 = ... (Kills %xmm0, all %xmm0s sub-registers, and %ymm0) /// -/// %YMM0<def> = ... -/// %XMM0<def> = ..., %YMM0<imp-use> (%YMM0 and all its sub-registers are alive) +/// %ymm0 = ... +/// %xmm0 = ..., implicit %ymm0 (%ymm0 and all its sub-registers are alive) //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_LIVEPHYSREGS_H @@ -32,8 +32,8 @@ #include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <utility> @@ -108,6 +108,12 @@ public: /// Returns true if register \p Reg and no aliasing register is in the set. bool available(const MachineRegisterInfo &MRI, unsigned Reg) const; + /// Remove defined registers and regmask kills from the set. + void removeDefs(const MachineInstr &MI); + + /// Add uses to the set. + void addUses(const MachineInstr &MI); + /// Simulates liveness when stepping backwards over an instruction(bundle). /// Remove Defs, add uses. This is the recommended way of calculating /// liveness. @@ -152,6 +158,10 @@ private: /// \brief Adds live-in registers from basic block \p MBB, taking associated /// lane masks into consideration. void addBlockLiveIns(const MachineBasicBlock &MBB); + + /// Adds pristine registers. Pristine registers are callee saved registers + /// that are unused in the function. + void addPristines(const MachineFunction &MF); }; inline raw_ostream &operator<<(raw_ostream &OS, const LivePhysRegs& LR) { @@ -159,12 +169,21 @@ inline raw_ostream &operator<<(raw_ostream &OS, const LivePhysRegs& LR) { return OS; } -/// \brief Computes the live-in list for \p MBB assuming all of its successors -/// live-in lists are up-to-date. Uses the given LivePhysReg instance \p -/// LiveRegs; This is just here to avoid repeated heap allocations when calling -/// this multiple times in a pass. -void computeLiveIns(LivePhysRegs &LiveRegs, const MachineRegisterInfo &MRI, - MachineBasicBlock &MBB); +/// \brief Computes registers live-in to \p MBB assuming all of its successors +/// live-in lists are up-to-date. Puts the result into the given LivePhysReg +/// instance \p LiveRegs. +void computeLiveIns(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB); + +/// Recomputes dead and kill flags in \p MBB. +void recomputeLivenessFlags(MachineBasicBlock &MBB); + +/// Adds registers contained in \p LiveRegs to the block live-in list of \p MBB. +/// Does not add reserved registers. +void addLiveIns(MachineBasicBlock &MBB, const LivePhysRegs &LiveRegs); + +/// Convenience function combining computeLiveIns() and addLiveIns(). +void computeAndAddLiveIns(LivePhysRegs &LiveRegs, + MachineBasicBlock &MBB); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h index 362d9854a271..84bccde0caa2 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveRangeEdit.h @@ -29,7 +29,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SlotIndexes.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include <cassert> namespace llvm { diff --git a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h index c28b1a06854f..dc4956da9637 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveRegUnits.h @@ -16,9 +16,9 @@ #define LLVM_CODEGEN_LIVEREGUNITS_H #include "llvm/ADT/BitVector.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/LaneBitmask.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cstdint> namespace llvm { @@ -51,7 +51,7 @@ public: void clear() { Units.reset(); } /// Returns true if the set is empty. - bool empty() const { return Units.empty(); } + bool empty() const { return Units.none(); } /// Adds register units covered by physical register \p Reg. void addReg(unsigned Reg) { @@ -123,6 +123,11 @@ public: const BitVector &getBitVector() const { return Units; } + +private: + /// Adds pristine registers. Pristine registers are callee saved registers + /// that are unused in the function. + void addPristines(const MachineFunction &MF); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h index d6e947c03dbd..ed8da8662106 100644 --- a/contrib/llvm/include/llvm/CodeGen/LiveVariables.h +++ b/contrib/llvm/include/llvm/CodeGen/LiveVariables.h @@ -36,7 +36,7 @@ #include "llvm/ADT/SparseBitVector.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" namespace llvm { diff --git a/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h index 1b1ba6a05837..ba40e522e261 100644 --- a/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/contrib/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -12,12 +12,18 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_CODEGEN_MIRYAMLMAPPING_H -#define LLVM_LIB_CODEGEN_MIRYAMLMAPPING_H +#ifndef LLVM_CODEGEN_MIRYAMLMAPPING_H +#define LLVM_CODEGEN_MIRYAMLMAPPING_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/Support/SMLoc.h" #include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cstdint> +#include <string> #include <vector> namespace llvm { @@ -29,7 +35,7 @@ struct StringValue { std::string Value; SMRange SourceRange; - StringValue() {} + StringValue() = default; StringValue(std::string Value) : Value(std::move(Value)) {} bool operator==(const StringValue &Other) const { @@ -38,7 +44,7 @@ struct StringValue { }; template <> struct ScalarTraits<StringValue> { - static void output(const StringValue &S, void *, llvm::raw_ostream &OS) { + static void output(const StringValue &S, void *, raw_ostream &OS) { OS << S.Value; } @@ -50,16 +56,16 @@ template <> struct ScalarTraits<StringValue> { return ""; } - static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); } + static QuotingType mustQuote(StringRef S) { return needsQuotes(S); } }; struct FlowStringValue : StringValue { - FlowStringValue() {} + FlowStringValue() = default; FlowStringValue(std::string Value) : StringValue(std::move(Value)) {} }; template <> struct ScalarTraits<FlowStringValue> { - static void output(const FlowStringValue &S, void *, llvm::raw_ostream &OS) { + static void output(const FlowStringValue &S, void *, raw_ostream &OS) { return ScalarTraits<StringValue>::output(S, nullptr, OS); } @@ -67,11 +73,12 @@ template <> struct ScalarTraits<FlowStringValue> { return ScalarTraits<StringValue>::input(Scalar, Ctx, S); } - static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); } + static QuotingType mustQuote(StringRef S) { return needsQuotes(S); } }; struct BlockStringValue { StringValue Value; + bool operator==(const BlockStringValue &Other) const { return Value == Other.Value; } @@ -90,10 +97,10 @@ template <> struct BlockScalarTraits<BlockStringValue> { /// A wrapper around unsigned which contains a source range that's being set /// during parsing. struct UnsignedValue { - unsigned Value; + unsigned Value = 0; SMRange SourceRange; - UnsignedValue() : Value(0) {} + UnsignedValue() = default; UnsignedValue(unsigned Value) : Value(Value) {} bool operator==(const UnsignedValue &Other) const { @@ -113,7 +120,7 @@ template <> struct ScalarTraits<UnsignedValue> { return ScalarTraits<unsigned>::input(Scalar, Ctx, Value.Value); } - static bool mustQuote(StringRef Scalar) { + static QuotingType mustQuote(StringRef Scalar) { return ScalarTraits<unsigned>::mustQuote(Scalar); } }; @@ -148,7 +155,9 @@ struct VirtualRegisterDefinition { UnsignedValue ID; StringValue Class; StringValue PreferredRegister; + // TODO: Serialize the target specific register hints. + bool operator==(const VirtualRegisterDefinition &Other) const { return ID == Other.ID && Class == Other.Class && PreferredRegister == Other.PreferredRegister; @@ -169,6 +178,7 @@ template <> struct MappingTraits<VirtualRegisterDefinition> { struct MachineFunctionLiveIn { StringValue Register; StringValue VirtualRegister; + bool operator==(const MachineFunctionLiveIn &Other) const { return Register == Other.Register && VirtualRegister == Other.VirtualRegister; @@ -202,16 +212,21 @@ struct MachineStackObject { int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; + uint8_t StackID = 0; StringValue CalleeSavedRegister; + bool CalleeSavedRestored = true; Optional<int64_t> LocalOffset; StringValue DebugVar; StringValue DebugExpr; StringValue DebugLoc; + bool operator==(const MachineStackObject &Other) const { return ID == Other.ID && Name == Other.Name && Type == Other.Type && Offset == Other.Offset && Size == Other.Size && Alignment == Other.Alignment && + StackID == Other.StackID && CalleeSavedRegister == Other.CalleeSavedRegister && + CalleeSavedRestored == Other.CalleeSavedRestored && LocalOffset == Other.LocalOffset && DebugVar == Other.DebugVar && DebugExpr == Other.DebugExpr && DebugLoc == Other.DebugLoc; } @@ -237,8 +252,11 @@ template <> struct MappingTraits<MachineStackObject> { if (Object.Type != MachineStackObject::VariableSized) YamlIO.mapRequired("size", Object.Size); YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0); + YamlIO.mapOptional("stack-id", Object.StackID); YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister, StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("callee-saved-restored", Object.CalleeSavedRestored, + true); YamlIO.mapOptional("local-offset", Object.LocalOffset, Optional<int64_t>()); YamlIO.mapOptional("di-variable", Object.DebugVar, StringValue()); // Don't print it out when it's empty. @@ -260,14 +278,19 @@ struct FixedMachineStackObject { int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; + uint8_t StackID = 0; bool IsImmutable = false; bool IsAliased = false; StringValue CalleeSavedRegister; + bool CalleeSavedRestored = true; + bool operator==(const FixedMachineStackObject &Other) const { return ID == Other.ID && Type == Other.Type && Offset == Other.Offset && Size == Other.Size && Alignment == Other.Alignment && + StackID == Other.StackID && IsImmutable == Other.IsImmutable && IsAliased == Other.IsAliased && - CalleeSavedRegister == Other.CalleeSavedRegister; + CalleeSavedRegister == Other.CalleeSavedRegister && + CalleeSavedRestored == Other.CalleeSavedRestored; } }; @@ -289,12 +312,15 @@ template <> struct MappingTraits<FixedMachineStackObject> { YamlIO.mapOptional("offset", Object.Offset, (int64_t)0); YamlIO.mapOptional("size", Object.Size, (uint64_t)0); YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0); + YamlIO.mapOptional("stack-id", Object.StackID); if (Object.Type != FixedMachineStackObject::SpillSlot) { YamlIO.mapOptional("isImmutable", Object.IsImmutable, false); YamlIO.mapOptional("isAliased", Object.IsAliased, false); } YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister, StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("callee-saved-restored", Object.CalleeSavedRestored, + true); } static const bool flow = true; @@ -304,9 +330,12 @@ struct MachineConstantPoolValue { UnsignedValue ID; StringValue Value; unsigned Alignment = 0; + bool IsTargetSpecific = false; + bool operator==(const MachineConstantPoolValue &Other) const { return ID == Other.ID && Value == Other.Value && - Alignment == Other.Alignment; + Alignment == Other.Alignment && + IsTargetSpecific == Other.IsTargetSpecific; } }; @@ -315,6 +344,7 @@ template <> struct MappingTraits<MachineConstantPoolValue> { YamlIO.mapRequired("id", Constant.ID); YamlIO.mapOptional("value", Constant.Value, StringValue()); YamlIO.mapOptional("alignment", Constant.Alignment, (unsigned)0); + YamlIO.mapOptional("isTargetSpecific", Constant.IsTargetSpecific, false); } }; @@ -322,6 +352,7 @@ struct MachineJumpTable { struct Entry { UnsignedValue ID; std::vector<FlowStringValue> Blocks; + bool operator==(const Entry &Other) const { return ID == Other.ID && Blocks == Other.Blocks; } @@ -329,6 +360,7 @@ struct MachineJumpTable { MachineJumpTableInfo::JTEntryKind Kind = MachineJumpTableInfo::EK_Custom32; std::vector<Entry> Entries; + bool operator==(const MachineJumpTable &Other) const { return Kind == Other.Kind && Entries == Other.Entries; } @@ -387,6 +419,7 @@ struct MachineFrameInfo { bool HasMustTailInVarArgFunc = false; StringValue SavePoint; StringValue RestorePoint; + bool operator==(const MachineFrameInfo &Other) const { return IsFrameAddressTaken == Other.IsFrameAddressTaken && IsReturnAddressTaken == Other.IsReturnAddressTaken && @@ -485,4 +518,4 @@ template <> struct MappingTraits<MachineFunction> { } // end namespace yaml } // end namespace llvm -#endif +#endif // LLVM_CODEGEN_MIRYAMLMAPPING_H diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h index 97a49ce4dc4f..0c9110cbaa87 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -25,6 +25,7 @@ #include "llvm/MC/LaneBitmask.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/BranchProbability.h" +#include "llvm/Support/Printable.h" #include <cassert> #include <cstdint> #include <functional> @@ -97,6 +98,8 @@ private: using const_probability_iterator = std::vector<BranchProbability>::const_iterator; + Optional<uint64_t> IrrLoopHeaderWeight; + /// Keep track of the physical registers that are livein of the basicblock. using LiveInVector = std::vector<RegisterMaskPair>; LiveInVector LiveIns; @@ -699,8 +702,8 @@ public: LQR_Unknown ///< Register liveness not decidable from local neighborhood. }; - /// Return whether (physical) register \p Reg has been <def>ined and not - /// <kill>ed as of just before \p Before. + /// Return whether (physical) register \p Reg has been defined and not + /// killed as of just before \p Before. /// /// Search is localised to a neighborhood of \p Neighborhood instructions /// before (searching for defs or kills) and \p Neighborhood instructions @@ -729,6 +732,14 @@ public: /// Return the MCSymbol for this basic block. MCSymbol *getSymbol() const; + Optional<uint64_t> getIrrLoopHeaderWeight() const { + return IrrLoopHeaderWeight; + } + + void setIrrLoopHeaderWeight(uint64_t Weight) { + IrrLoopHeaderWeight = Weight; + } + private: /// Return probability iterator corresponding to the I successor iterator. probability_iterator getProbabilityIterator(succ_iterator I); @@ -748,7 +759,7 @@ private: // Machine-CFG mutators - /// Remove Pred as a predecessor of this MachineBasicBlock. Don't do this + /// Add Pred as a predecessor of this MachineBasicBlock. Don't do this /// unless you know what you're doing, because it doesn't update Pred's /// successors list. Use Pred->addSuccessor instead. void addPredecessor(MachineBasicBlock *Pred); @@ -761,9 +772,17 @@ private: raw_ostream& operator<<(raw_ostream &OS, const MachineBasicBlock &MBB); +/// Prints a machine basic block reference. +/// +/// The format is: +/// %bb.5 - a machine basic block with MBB.getNumber() == 5. +/// +/// Usage: OS << printMBBReference(MBB) << '\n'; +Printable printMBBReference(const MachineBasicBlock &MBB); + // This is useful when building IndexedMaps keyed on basic block pointers. -struct MBB2NumberFunctor : - public std::unary_function<const MachineBasicBlock*, unsigned> { +struct MBB2NumberFunctor { + using argument_type = const MachineBasicBlock *; unsigned operator()(const MachineBasicBlock *MBB) const { return MBB->getNumber(); } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index cba79c818a76..5b4b99ca0a5d 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -62,6 +62,8 @@ public: Optional<uint64_t> getBlockProfileCount(const MachineBasicBlock *MBB) const; Optional<uint64_t> getProfileCountFromFreq(uint64_t Freq) const; + bool isIrrLoopHeader(const MachineBasicBlock *MBB); + const MachineFunction *getFunction() const; const MachineBranchProbabilityInfo *getMBPI() const; void view(const Twine &Name, bool isSimple = true) const; diff --git a/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h index 8c54ae925470..586535f771c2 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineCombinerPattern.h @@ -68,12 +68,18 @@ enum class MachineCombinerPattern { FMLAv4i32_indexed_OP2, FMLSv1i32_indexed_OP2, FMLSv1i64_indexed_OP2, - FMLSv2i32_indexed_OP2, - FMLSv2i64_indexed_OP2, + FMLSv2f32_OP1, FMLSv2f32_OP2, + FMLSv2f64_OP1, FMLSv2f64_OP2, - FMLSv4i32_indexed_OP2, - FMLSv4f32_OP2 + FMLSv2i32_indexed_OP1, + FMLSv2i32_indexed_OP2, + FMLSv2i64_indexed_OP1, + FMLSv2i64_indexed_OP2, + FMLSv4f32_OP1, + FMLSv4f32_OP2, + FMLSv4i32_indexed_OP1, + FMLSv4i32_indexed_OP2 }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/MachineDominanceFrontier.h b/contrib/llvm/include/llvm/CodeGen/MachineDominanceFrontier.h index 6efeefd9a721..ffbcc62bfa36 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineDominanceFrontier.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineDominanceFrontier.h @@ -39,7 +39,7 @@ public: DominanceFrontierBase<MachineBasicBlock, false> &getBase() { return Base; } - inline const std::vector<MachineBasicBlock *> &getRoots() const { + const SmallVectorImpl<MachineBasicBlock *> &getRoots() const { return Base.getRoots(); } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineDominators.h b/contrib/llvm/include/llvm/CodeGen/MachineDominators.h index 8bf98f606495..98fdb51aae2f 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineDominators.h @@ -93,7 +93,7 @@ public: /// multiple blocks if we are computing post dominators. For forward /// dominators, this will always be a single block (the entry node). /// - inline const std::vector<MachineBasicBlock*> &getRoots() const { + inline const SmallVectorImpl<MachineBasicBlock*> &getRoots() const { applySplitCriticalEdges(); return DT->getRoots(); } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h index 689f3cd9fd12..f887517217e1 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -31,15 +31,30 @@ class AllocaInst; class CalleeSavedInfo { unsigned Reg; int FrameIdx; + /// Flag indicating whether the register is actually restored in the epilog. + /// In most cases, if a register is saved, it is also restored. There are + /// some situations, though, when this is not the case. For example, the + /// LR register on ARM is usually saved, but on exit from the function its + /// saved value may be loaded directly into PC. Since liveness tracking of + /// physical registers treats callee-saved registers are live outside of + /// the function, LR would be treated as live-on-exit, even though in these + /// scenarios it is not. This flag is added to indicate that the saved + /// register described by this object is not restored in the epilog. + /// The long-term solution is to model the liveness of callee-saved registers + /// by implicit uses on the return instructions, however, the required + /// changes in the ARM backend would be quite extensive. + bool Restored; public: explicit CalleeSavedInfo(unsigned R, int FI = 0) - : Reg(R), FrameIdx(FI) {} + : Reg(R), FrameIdx(FI), Restored(true) {} // Accessors. unsigned getReg() const { return Reg; } int getFrameIdx() const { return FrameIdx; } void setFrameIdx(int FI) { FrameIdx = FI; } + bool isRestored() const { return Restored; } + void setRestored(bool R) { Restored = R; } }; /// The MachineFrameInfo class represents an abstract stack frame until @@ -99,8 +114,16 @@ class MachineFrameInfo { /// and/or GC related) over a statepoint. We know that the address of the /// slot can't alias any LLVM IR value. This is very similar to a Spill /// Slot, but is created by statepoint lowering is SelectionDAG, not the - /// register allocator. - bool isStatepointSpillSlot; + /// register allocator. + bool isStatepointSpillSlot = false; + + /// Identifier for stack memory type analagous to address space. If this is + /// non-0, the meaning is target defined. Offsets cannot be directly + /// compared between objects with different stack IDs. The object may not + /// necessarily reside in the same contiguous memory block as other stack + /// objects. Objects with differing stack IDs should not be merged or + /// replaced substituted for each other. + uint8_t StackID; /// If this stack object is originated from an Alloca instruction /// this value saves the original IR allocation. Can be NULL. @@ -108,7 +131,7 @@ class MachineFrameInfo { // If true, the object was mapped into the local frame // block and doesn't need additional handling for allocation beyond that. - bool PreAllocated; + bool PreAllocated = false; // If true, an LLVM IR value might point to this object. // Normally, spill slots and fixed-offset objects don't alias IR-accessible @@ -117,16 +140,17 @@ class MachineFrameInfo { bool isAliased; /// If true, the object has been zero-extended. - bool isZExt; + bool isZExt = false; /// If true, the object has been zero-extended. - bool isSExt; - - StackObject(uint64_t Sz, unsigned Al, int64_t SP, bool IM, - bool isSS, const AllocaInst *Val, bool A) - : SPOffset(SP), Size(Sz), Alignment(Al), isImmutable(IM), - isSpillSlot(isSS), isStatepointSpillSlot(false), Alloca(Val), - PreAllocated(false), isAliased(A), isZExt(false), isSExt(false) {} + bool isSExt = false; + + StackObject(uint64_t Size, unsigned Alignment, int64_t SPOffset, + bool IsImmutable, bool IsSpillSlot, const AllocaInst *Alloca, + bool IsAliased, uint8_t StackID = 0) + : SPOffset(SPOffset), Size(Size), Alignment(Alignment), + isImmutable(IsImmutable), isSpillSlot(IsSpillSlot), + StackID(StackID), Alloca(Alloca), isAliased(IsAliased) {} }; /// The alignment of the stack. @@ -549,13 +573,13 @@ public: /// All fixed objects should be created before other objects are created for /// efficiency. By default, fixed objects are not pointed to by LLVM IR /// values. This returns an index with a negative value. - int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable, + int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased = false); /// Create a spill slot at a fixed location on the stack. /// Returns an index with a negative value. int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset, - bool Immutable = false); + bool IsImmutable = false); /// Returns true if the specified index corresponds to a fixed stack object. bool isFixedObjectIndex(int ObjectIdx) const { @@ -581,10 +605,10 @@ public: } /// Marks the immutability of an object. - void setIsImmutableObjectIndex(int ObjectIdx, bool Immutable) { + void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable) { assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); - Objects[ObjectIdx+NumFixedObjects].isImmutable = Immutable; + Objects[ObjectIdx+NumFixedObjects].isImmutable = IsImmutable; } /// Returns true if the specified index corresponds to a spill slot. @@ -600,6 +624,18 @@ public: return Objects[ObjectIdx+NumFixedObjects].isStatepointSpillSlot; } + /// \see StackID + uint8_t getStackID(int ObjectIdx) const { + return Objects[ObjectIdx+NumFixedObjects].StackID; + } + + /// \see StackID + void setStackID(int ObjectIdx, uint8_t ID) { + assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && + "Invalid Object Idx!"); + Objects[ObjectIdx+NumFixedObjects].StackID = ID; + } + /// Returns true if the specified index corresponds to a dead object. bool isDeadObjectIndex(int ObjectIdx) const { assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && @@ -624,8 +660,8 @@ public: /// Create a new statically sized stack object, returning /// a nonnegative identifier to represent it. - int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, - const AllocaInst *Alloca = nullptr); + int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, + const AllocaInst *Alloca = nullptr, uint8_t ID = 0); /// Create a new statically sized stack object that represents a spill slot, /// returning a nonnegative identifier to represent it. @@ -646,6 +682,8 @@ public: const std::vector<CalleeSavedInfo> &getCalleeSavedInfo() const { return CSInfo; } + /// \copydoc getCalleeSavedInfo() + std::vector<CalleeSavedInfo> &getCalleeSavedInfo() { return CSInfo; } /// Used by prolog/epilog inserter to set the function's callee saved /// information. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h index 010d7032c516..7d8b7ebe8d62 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineFunction.h @@ -223,7 +223,7 @@ struct LandingPadInfo { }; class MachineFunction { - const Function *Fn; + const Function &F; const TargetMachine &Target; const TargetSubtargetInfo *STI; MCContext &Ctx; @@ -314,6 +314,9 @@ class MachineFunction { /// Map of invoke call site index values to associated begin EH_LABEL. DenseMap<MCSymbol*, unsigned> CallSiteMap; + /// CodeView label annotations. + std::vector<std::pair<MCSymbol *, MDNode *>> CodeViewAnnotations; + bool CallsEHReturn = false; bool CallsUnwindInit = false; bool HasEHFunclets = false; @@ -356,8 +359,9 @@ public: using VariableDbgInfoMapTy = SmallVector<VariableDbgInfo, 4>; VariableDbgInfoMapTy VariableDbgInfos; - MachineFunction(const Function *Fn, const TargetMachine &TM, - unsigned FunctionNum, MachineModuleInfo &MMI); + MachineFunction(const Function &F, const TargetMachine &TM, + const TargetSubtargetInfo &STI, unsigned FunctionNum, + MachineModuleInfo &MMI); MachineFunction(const MachineFunction &) = delete; MachineFunction &operator=(const MachineFunction &) = delete; ~MachineFunction(); @@ -376,8 +380,8 @@ public: /// Return the DataLayout attached to the Module associated to this MF. const DataLayout &getDataLayout() const; - /// getFunction - Return the LLVM function that this machine code represents - const Function *getFunction() const { return Fn; } + /// Return the LLVM function that this machine code represents + const Function &getFunction() const { return F; } /// getName - Return the name of the corresponding LLVM function. StringRef getName() const; @@ -625,14 +629,23 @@ public: MachineInstr *CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, bool NoImp = false); - /// CloneMachineInstr - Create a new MachineInstr which is a copy of the - /// 'Orig' instruction, identical in all ways except the instruction - /// has no parent, prev, or next. + /// Create a new MachineInstr which is a copy of \p Orig, identical in all + /// ways except the instruction has no parent, prev, or next. Bundling flags + /// are reset. /// - /// See also TargetInstrInfo::duplicate() for target-specific fixes to cloned - /// instructions. + /// Note: Clones a single instruction, not whole instruction bundles. + /// Does not perform target specific adjustments; consider using + /// TargetInstrInfo::duplicate() instead. MachineInstr *CloneMachineInstr(const MachineInstr *Orig); + /// Clones instruction or the whole instruction bundle \p Orig and insert + /// into \p MBB before \p InsertBefore. + /// + /// Note: Does not perform target specific adjustments; consider using + /// TargetInstrInfo::duplicate() intead. + MachineInstr &CloneMachineInstrBundle(MachineBasicBlock &MBB, + MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig); + /// DeleteMachineInstr - Delete the given MachineInstr. void DeleteMachineInstr(MachineInstr *MI); @@ -823,6 +836,15 @@ public: return CallSiteMap.count(BeginLabel); } + /// Record annotations associated with a particular label. + void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD) { + CodeViewAnnotations.push_back({Label, MD}); + } + + ArrayRef<std::pair<MCSymbol *, MDNode *>> getCodeViewAnnotations() const { + return CodeViewAnnotations; + } + /// Return a reference to the C++ typeinfo for the current function. const std::vector<const GlobalValue *> &getTypeInfos() const { return TypeInfos; diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h index b87aff102d47..3c1c1bb14f42 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstr.h @@ -22,11 +22,11 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/ArrayRecycler.h" -#include "llvm/Target/TargetOpcodes.h" #include <algorithm> #include <cassert> #include <cstdint> @@ -44,6 +44,7 @@ class MachineRegisterInfo; class ModuleSlotTracker; class raw_ostream; template <typename T> class SmallVectorImpl; +class SmallBitVector; class StringRef; class TargetInstrInfo; class TargetRegisterClass; @@ -67,7 +68,9 @@ public: /// otherwise easily derivable from the IR text. /// enum CommentFlag { - ReloadReuse = 0x1 // higher bits are reserved for target dep comments. + ReloadReuse = 0x1, // higher bits are reserved for target dep comments. + NoSchedComment = 0x2, + TAsmComments = 0x4 // Target Asm comments should start from this value. }; enum MIFlag { @@ -139,6 +142,17 @@ public: const MachineBasicBlock* getParent() const { return Parent; } MachineBasicBlock* getParent() { return Parent; } + /// Return the function that contains the basic block that this instruction + /// belongs to. + /// + /// Note: this is undefined behaviour if the instruction does not have a + /// parent. + const MachineFunction *getMF() const; + MachineFunction *getMF() { + return const_cast<MachineFunction *>( + static_cast<const MachineInstr *>(this)->getMF()); + } + /// Return the asm printer flags bitvector. uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } @@ -290,6 +304,21 @@ public: return Operands[i]; } + /// Return true if operand \p OpIdx is a subregister index. + bool isOperandSubregIdx(unsigned OpIdx) const { + assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && + "Expected MO_Immediate operand type."); + if (isExtractSubreg() && OpIdx == 2) + return true; + if (isInsertSubreg() && OpIdx == 3) + return true; + if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) + return true; + if (isSubregToReg() && OpIdx == 3) + return true; + return false; + } + /// Returns the number of non-implicit operands. unsigned getNumExplicitOperands() const; @@ -771,9 +800,14 @@ public: bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } + bool isAnnotationLabel() const { + return getOpcode() == TargetOpcode::ANNOTATION_LABEL; + } /// Returns true if the MachineInstr represents a label. - bool isLabel() const { return isEHLabel() || isGCLabel(); } + bool isLabel() const { + return isEHLabel() || isGCLabel() || isAnnotationLabel(); + } bool isCFIInstruction() const { return getOpcode() == TargetOpcode::CFI_INSTRUCTION; @@ -792,7 +826,10 @@ public: && getOperand(1).isImm(); } - bool isPHI() const { return getOpcode() == TargetOpcode::PHI; } + bool isPHI() const { + return getOpcode() == TargetOpcode::PHI || + getOpcode() == TargetOpcode::G_PHI; + } bool isKill() const { return getOpcode() == TargetOpcode::KILL; } bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } bool isInlineAsm() const { return getOpcode() == TargetOpcode::INLINEASM; } @@ -869,6 +906,7 @@ public: return isMetaInstruction(); // Copy-like instructions are usually eliminated during register allocation. case TargetOpcode::PHI: + case TargetOpcode::G_PHI: case TargetOpcode::COPY: case TargetOpcode::INSERT_SUBREG: case TargetOpcode::SUBREG_TO_REG: @@ -1185,6 +1223,15 @@ public: /// Debugging support /// @{ + /// Determine the generic type to be printed (if needed) on uses and defs. + LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, + const MachineRegisterInfo &MRI) const; + + /// Return true when an instruction has tied register that can't be determined + /// by the instruction's descriptor. This is useful for MIR printing, to + /// determine whether we need to print the ties or not. + bool hasComplexRegisterTies() const; + /// Print this MI to \p OS. /// Only print the defs and the opcode if \p SkipOpers is true. /// Otherwise, also print operands if \p SkipDebugLoc is true. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 412c55d542ea..e4f3976ec950 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Support/ErrorHandling.h" @@ -48,6 +49,7 @@ namespace RegState { EarlyClobber = 0x40, Debug = 0x80, InternalRead = 0x100, + Renamable = 0x200, DefineNoRead = Define | Undef, ImplicitDefine = Implicit | Define, ImplicitKill = Implicit | Kill @@ -91,7 +93,8 @@ public: flags & RegState::EarlyClobber, SubReg, flags & RegState::Debug, - flags & RegState::InternalRead)); + flags & RegState::InternalRead, + flags & RegState::Renamable)); return *this; } @@ -396,28 +399,32 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, const DebugLoc &DL, } /// This version of the builder builds a DBG_VALUE intrinsic -/// for either a value in a register or a register-indirect+offset +/// for either a value in a register or a register-indirect /// address. The convention is that a DBG_VALUE is indirect iff the /// second operand is an immediate. MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID, bool IsIndirect, - unsigned Reg, unsigned Offset, - const MDNode *Variable, const MDNode *Expr); + unsigned Reg, const MDNode *Variable, + const MDNode *Expr); /// This version of the builder builds a DBG_VALUE intrinsic -/// for either a value in a register or a register-indirect+offset +/// for either a value in a register or a register-indirect /// address and inserts it at position I. MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, const DebugLoc &DL, const MCInstrDesc &MCID, bool IsIndirect, - unsigned Reg, unsigned Offset, - const MDNode *Variable, const MDNode *Expr); + unsigned Reg, const MDNode *Variable, + const MDNode *Expr); /// Clone a DBG_VALUE whose value has been spilled to FrameIndex. MachineInstr *buildDbgValueForSpill(MachineBasicBlock &BB, MachineBasicBlock::iterator I, const MachineInstr &Orig, int FrameIndex); +/// Update a DBG_VALUE whose value has been spilled to FrameIndex. Useful when +/// modifying an instruction in place while iterating over a basic block. +void updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex); + inline unsigned getDefRegState(bool B) { return B ? RegState::Define : 0; } @@ -439,6 +446,9 @@ inline unsigned getInternalReadRegState(bool B) { inline unsigned getDebugRegState(bool B) { return B ? RegState::Debug : 0; } +inline unsigned getRenamableRegState(bool B) { + return B ? RegState::Renamable : 0; +} /// Get all register state flags from machine operand \p RegOp. inline unsigned getRegState(const MachineOperand &RegOp) { @@ -449,7 +459,10 @@ inline unsigned getRegState(const MachineOperand &RegOp) { getDeadRegState(RegOp.isDead()) | getUndefRegState(RegOp.isUndef()) | getInternalReadRegState(RegOp.isInternalRead()) | - getDebugRegState(RegOp.isDebug()); + getDebugRegState(RegOp.isDebug()) | + getRenamableRegState( + TargetRegisterInfo::isPhysicalRegister(RegOp.getReg()) && + RegOp.isRenamable()); } /// Helper class for constructing bundles of MachineInstrs. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineInstrBundle.h b/contrib/llvm/include/llvm/CodeGen/MachineInstrBundle.h index 995c7001d928..b5341fd1ae49 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineInstrBundle.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineInstrBundle.h @@ -150,7 +150,7 @@ public: /// struct VirtRegInfo { /// Reads - One of the operands read the virtual register. This does not - /// include <undef> or <internal> use operands, see MO::readsReg(). + /// include undef or internal use operands, see MO::readsReg(). bool Reads; /// Writes - One of the operands writes the virtual register. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h index adcd1d0de63d..25a3e6b556a3 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h @@ -20,6 +20,7 @@ #ifndef LLVM_CODEGEN_MACHINEJUMPTABLEINFO_H #define LLVM_CODEGEN_MACHINEJUMPTABLEINFO_H +#include "llvm/Support/Printable.h" #include <cassert> #include <vector> @@ -125,6 +126,15 @@ public: void dump() const; }; + +/// Prints a jump table entry reference. +/// +/// The format is: +/// %jump-table.5 - a jump table entry with index == 5. +/// +/// Usage: OS << printJumpTableEntryReference(Idx) << '\n'; +Printable printJumpTableEntryReference(unsigned Idx); + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineLoopInfo.h index 58cffaade9d2..104655e45524 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineLoopInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineLoopInfo.h @@ -44,8 +44,6 @@ extern template class LoopBase<MachineBasicBlock, MachineLoop>; class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> { public: - MachineLoop(); - /// Return the "top" block in the loop, which is the first block in the linear /// layout, ignoring any parts of the loop not contiguous with the part that /// contains the header. @@ -76,6 +74,8 @@ private: explicit MachineLoop(MachineBasicBlock *MBB) : LoopBase<MachineBasicBlock, MachineLoop>(MBB) {} + + MachineLoop() = default; }; // Implementation in LoopInfoImpl.h diff --git a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h index a9de0db05d72..c5b204a79f04 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -45,18 +45,46 @@ struct MachinePointerInfo { /// Offset - This is an offset from the base Value*. int64_t Offset; - explicit MachinePointerInfo(const Value *v = nullptr, int64_t offset = 0) - : V(v), Offset(offset) {} + uint8_t StackID; - explicit MachinePointerInfo(const PseudoSourceValue *v, - int64_t offset = 0) - : V(v), Offset(offset) {} + unsigned AddrSpace = 0; + + explicit MachinePointerInfo(const Value *v, int64_t offset = 0, + uint8_t ID = 0) + : V(v), Offset(offset), StackID(ID) { + AddrSpace = v ? v->getType()->getPointerAddressSpace() : 0; + } + + explicit MachinePointerInfo(const PseudoSourceValue *v, int64_t offset = 0, + uint8_t ID = 0) + : V(v), Offset(offset), StackID(ID) { + AddrSpace = v ? v->getAddressSpace() : 0; + } + + explicit MachinePointerInfo(unsigned AddressSpace = 0) + : V((const Value *)nullptr), Offset(0), StackID(0), + AddrSpace(AddressSpace) {} + + explicit MachinePointerInfo( + PointerUnion<const Value *, const PseudoSourceValue *> v, + int64_t offset = 0, + uint8_t ID = 0) + : V(v), Offset(offset), StackID(ID) { + if (V) { + if (const auto *ValPtr = V.dyn_cast<const Value*>()) + AddrSpace = ValPtr->getType()->getPointerAddressSpace(); + else + AddrSpace = V.get<const PseudoSourceValue*>()->getAddressSpace(); + } + } MachinePointerInfo getWithOffset(int64_t O) const { - if (V.isNull()) return MachinePointerInfo(); + if (V.isNull()) + return MachinePointerInfo(AddrSpace); if (V.is<const Value*>()) - return MachinePointerInfo(V.get<const Value*>(), Offset+O); - return MachinePointerInfo(V.get<const PseudoSourceValue*>(), Offset+O); + return MachinePointerInfo(V.get<const Value*>(), Offset+O, StackID); + return MachinePointerInfo(V.get<const PseudoSourceValue*>(), Offset+O, + StackID); } /// Return true if memory region [V, V+Offset+Size) is known to be @@ -82,7 +110,11 @@ struct MachinePointerInfo { static MachinePointerInfo getGOT(MachineFunction &MF); /// Stack pointer relative access. - static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset); + static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, + uint8_t ID = 0); + + /// Stack memory without other information. + static MachinePointerInfo getUnknownStack(MachineFunction &MF); }; diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h index d64941a9e725..6be304fa368b 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfo.h @@ -125,6 +125,16 @@ class MachineModuleInfo : public ImmutablePass { /// comments in lib/Target/X86/X86FrameLowering.cpp for more details. bool UsesMorestackAddr; + /// True if the module contains split-stack functions. This is used to + /// emit .note.GNU-split-stack section as required by the linker for + /// special handling split-stack function calling no-split-stack function. + bool HasSplitStack; + + /// True if the module contains no-split-stack functions. This is used to + /// emit .note.GNU-no-split-stack section when it also contains split-stack + /// functions. + bool HasNosplitStack; + /// Maps IR Functions to their corresponding MachineFunctions. DenseMap<const Function*, std::unique_ptr<MachineFunction>> MachineFunctions; /// Next unique number available for a MachineFunction. @@ -145,7 +155,6 @@ public: const MCContext &getContext() const { return Context; } MCContext &getContext() { return Context; } - void setModule(const Module *M) { TheModule = M; } const Module *getModule() const { return TheModule; } /// Returns the MachineFunction constructed for the IR function \p F. @@ -194,6 +203,22 @@ public: UsesMorestackAddr = b; } + bool hasSplitStack() const { + return HasSplitStack; + } + + void setHasSplitStack(bool b) { + HasSplitStack = b; + } + + bool hasNosplitStack() const { + return HasNosplitStack; + } + + void setHasNosplitStack(bool b) { + HasNosplitStack = b; + } + /// Return the symbol to be used for the specified basic block when its /// address is taken. This cannot be its normal LBB label because the block /// may be accessed outside its containing function. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h index 34b21ceddd43..6a87fa2fbf00 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/MachineModuleInfoImpls.h -------------------*- C++ -*-===// +//===- llvm/CodeGen/MachineModuleInfoImpls.h --------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,11 +15,12 @@ #ifndef LLVM_CODEGEN_MACHINEMODULEINFOIMPLS_H #define LLVM_CODEGEN_MACHINEMODULEINFOIMPLS_H -#include "llvm/BinaryFormat/Wasm.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/ValueTypes.h" +#include <cassert> namespace llvm { + class MCSymbol; /// MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation @@ -36,6 +37,7 @@ class MachineModuleInfoMachO : public MachineModuleInfoImpl { DenseMap<MCSymbol *, StubValueTy> ThreadLocalGVStubs; virtual void anchor(); // Out of line virtual method. + public: MachineModuleInfoMachO(const MachineModuleInfo &) {} @@ -64,6 +66,7 @@ class MachineModuleInfoELF : public MachineModuleInfoImpl { DenseMap<MCSymbol *, StubValueTy> GVStubs; virtual void anchor(); // Out of line virtual method. + public: MachineModuleInfoELF(const MachineModuleInfo &) {} @@ -79,4 +82,4 @@ public: } // end namespace llvm -#endif +#endif // LLVM_CODEGEN_MACHINEMODULEINFOIMPLS_H diff --git a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h index 2560399bcf54..ccf0917ed085 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineOperand.h @@ -14,8 +14,10 @@ #ifndef LLVM_CODEGEN_MACHINEOPERAND_H #define LLVM_CODEGEN_MACHINEOPERAND_H +#include "llvm/ADT/DenseMap.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/LowLevelTypeImpl.h" #include <cassert> namespace llvm { @@ -65,6 +67,7 @@ public: MO_CFIIndex, ///< MCCFIInstruction index. MO_IntrinsicID, ///< Intrinsic ID for ISel MO_Predicate, ///< Generic predicate for ISel + MO_Last = MO_Predicate, }; private: @@ -83,24 +86,30 @@ private: /// before MachineInstr::tieOperands(). unsigned char TiedTo : 4; - /// IsDef/IsImp/IsKill/IsDead flags - These are only valid for MO_Register - /// operands. - /// IsDef - True if this is a def, false if this is a use of the register. + /// This is only valid on register operands. /// bool IsDef : 1; /// IsImp - True if this is an implicit def or use, false if it is explicit. + /// This is only valid on register opderands. /// bool IsImp : 1; - /// IsKill - True if this instruction is the last use of the register on this - /// path through the function. This is only valid on uses of registers. - bool IsKill : 1; - - /// IsDead - True if this register is never used by a subsequent instruction. - /// This is only valid on definitions of registers. - bool IsDead : 1; + /// IsDeadOrKill + /// For uses: IsKill - True if this instruction is the last use of the + /// register on this path through the function. + /// For defs: IsDead - True if this register is never used by a subsequent + /// instruction. + /// This is only valid on register operands. + bool IsDeadOrKill : 1; + + /// IsRenamable - True if this register may be renamed, i.e. it does not + /// generate a value that is somehow read in a way that is not represented by + /// the Machine IR (e.g. to meet an ABI or ISA requirement). This is only + /// valid on physical register operands. Virtual registers are assumed to + /// always be renamable regardless of the value of this field. + bool IsRenamable : 1; /// IsUndef - True if this register operand reads an "undef" value, i.e. the /// read value doesn't matter. This flag can be set on both use and def @@ -114,9 +123,9 @@ private: /// the same register. In that case, the instruction may depend on those /// operands reading the same dont-care value. For example: /// - /// %vreg1<def> = XOR %vreg2<undef>, %vreg2<undef> + /// %1 = XOR undef %2, undef %2 /// - /// Any register can be used for %vreg2, and its value doesn't matter, but + /// Any register can be used for %2, and its value doesn't matter, but /// the two operands must be the same register. /// bool IsUndef : 1; @@ -224,11 +233,50 @@ public: /// void clearParent() { ParentMI = nullptr; } + /// Print a subreg index operand. + /// MO_Immediate operands can also be subreg idices. If it's the case, the + /// subreg index name will be printed. MachineInstr::isOperandSubregIdx can be + /// called to check this. + static void printSubregIdx(raw_ostream &OS, uint64_t Index, + const TargetRegisterInfo *TRI); + + /// Print operand target flags. + static void printTargetFlags(raw_ostream& OS, const MachineOperand &Op); + + /// Print a MCSymbol as an operand. + static void printSymbol(raw_ostream &OS, MCSymbol &Sym); + + /// Print a stack object reference. + static void printStackObjectReference(raw_ostream &OS, unsigned FrameIndex, + bool IsFixed, StringRef Name); + + /// Print the MachineOperand to \p os. + /// Providing a valid \p TRI and \p IntrinsicInfo results in a more + /// target-specific printing. If \p TRI and \p IntrinsicInfo are null, the + /// function will try to pick it up from the parent. void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr, const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const; - void print(raw_ostream &os, ModuleSlotTracker &MST, - const TargetRegisterInfo *TRI = nullptr, - const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const; + + /// More complex way of printing a MachineOperand. + /// \param TypeToPrint specifies the generic type to be printed on uses and + /// defs. It can be determined using MachineInstr::getTypeToPrint. + /// \param PrintDef - whether we want to print `def` on an operand which + /// isDef. Sometimes, if the operand is printed before '=', we don't print + /// `def`. + /// \param ShouldPrintRegisterTies - whether we want to print register ties. + /// Sometimes they are easily determined by the instruction's descriptor + /// (MachineInstr::hasComplexRegiterTies can determine if it's needed). + /// \param TiedOperandIdx - if we need to print register ties this needs to + /// provide the index of the tied register. If not, it will be ignored. + /// \param TRI - provide more target-specific information to the printer. + /// Unlike the previous function, this one will not try and get the + /// information from it's parent. + /// \param IntrinsicInfo - same as \p TRI. + void print(raw_ostream &os, ModuleSlotTracker &MST, LLT TypeToPrint, + bool PrintDef, bool ShouldPrintRegisterTies, + unsigned TiedOperandIdx, const TargetRegisterInfo *TRI, + const TargetIntrinsicInfo *IntrinsicInfo) const; + void dump() const; //===--------------------------------------------------------------------===// @@ -301,12 +349,12 @@ public: bool isDead() const { assert(isReg() && "Wrong MachineOperand accessor"); - return IsDead; + return IsDeadOrKill & IsDef; } bool isKill() const { assert(isReg() && "Wrong MachineOperand accessor"); - return IsKill; + return IsDeadOrKill & !IsDef; } bool isUndef() const { @@ -314,6 +362,8 @@ public: return IsUndef; } + bool isRenamable() const; + bool isInternalRead() const { assert(isReg() && "Wrong MachineOperand accessor"); return IsInternalRead; @@ -369,12 +419,13 @@ public: /// substPhysReg - Substitute the current register with the physical register /// Reg, taking any existing SubReg into account. For instance, - /// substPhysReg(%EAX) will change %reg1024:sub_8bit to %AL. + /// substPhysReg(%eax) will change %reg1024:sub_8bit to %al. /// void substPhysReg(unsigned Reg, const TargetRegisterInfo&); void setIsUse(bool Val = true) { setIsDef(!Val); } + /// Change a def to a use, or a use to a def. void setIsDef(bool Val = true); void setImplicit(bool Val = true) { @@ -385,12 +436,12 @@ public: void setIsKill(bool Val = true) { assert(isReg() && !IsDef && "Wrong MachineOperand mutator"); assert((!Val || !isDebug()) && "Marking a debug operation as kill"); - IsKill = Val; + IsDeadOrKill = Val; } void setIsDead(bool Val = true) { assert(isReg() && IsDef && "Wrong MachineOperand mutator"); - IsDead = Val; + IsDeadOrKill = Val; } void setIsUndef(bool Val = true) { @@ -398,6 +449,12 @@ public: IsUndef = Val; } + void setIsRenamable(bool Val = true); + + /// Set IsRenamable to true if there are no extra register allocation + /// requirements placed on this operand by the parent instruction's opcode. + void setIsRenamableIfNoExtraRegAllocReq(); + void setIsInternalRead(bool Val = true) { assert(isReg() && "Wrong MachineOperand mutator"); IsInternalRead = Val; @@ -549,6 +606,11 @@ public: Contents.OffsetedInfo.Val.Index = Idx; } + void setMetadata(const MDNode *MD) { + assert(isMetadata() && "Wrong MachineOperand mutator"); + Contents.MD = MD; + } + void setMBB(MachineBasicBlock *MBB) { assert(isMBB() && "Wrong MachineOperand mutator"); Contents.MBB = MBB; @@ -568,14 +630,16 @@ public: //===--------------------------------------------------------------------===// /// Returns true if this operand is identical to the specified operand except - /// for liveness related flags (isKill, isUndef and isDead). + /// for liveness related flags (isKill, isUndef and isDead). Note that this + /// should stay in sync with the hash_value overload below. bool isIdenticalTo(const MachineOperand &Other) const; /// \brief MachineOperand hash_value overload. /// /// Note that this includes the same information in the hash that /// isIdenticalTo uses for comparison. It is thus suited for use in hash - /// tables which use that function for equality comparisons only. + /// tables which use that function for equality comparisons only. This must + /// stay exactly in sync with isIdenticalTo above. friend hash_code hash_value(const MachineOperand &MO); /// ChangeToImmediate - Replace this operand with a new immediate operand of @@ -597,6 +661,10 @@ public: /// Replace this operand with a frame index. void ChangeToFrameIndex(int Idx); + /// Replace this operand with a target index. + void ChangeToTargetIndex(unsigned Idx, int64_t Offset, + unsigned char TargetFlags = 0); + /// ChangeToRegister - Replace this operand with a new register operand of /// the specified value. If an operand is known to be an register already, /// the setReg method should be used. @@ -630,16 +698,16 @@ public: bool isKill = false, bool isDead = false, bool isUndef = false, bool isEarlyClobber = false, - unsigned SubReg = 0, - bool isDebug = false, - bool isInternalRead = false) { + unsigned SubReg = 0, bool isDebug = false, + bool isInternalRead = false, + bool isRenamable = false) { assert(!(isDead && !isDef) && "Dead flag on non-def"); assert(!(isKill && isDef) && "Kill flag on def"); MachineOperand Op(MachineOperand::MO_Register); Op.IsDef = isDef; Op.IsImp = isImp; - Op.IsKill = isKill; - Op.IsDead = isDead; + Op.IsDeadOrKill = isKill | isDead; + Op.IsRenamable = isRenamable; Op.IsUndef = isUndef; Op.IsInternalRead = isInternalRead; Op.IsEarlyClobber = isEarlyClobber; @@ -679,8 +747,7 @@ public: Op.setTargetFlags(TargetFlags); return Op; } - static MachineOperand CreateJTI(unsigned Idx, - unsigned char TargetFlags = 0) { + static MachineOperand CreateJTI(unsigned Idx, unsigned char TargetFlags = 0) { MachineOperand Op(MachineOperand::MO_JumpTableIndex); Op.setIndex(Idx); Op.setTargetFlags(TargetFlags); @@ -711,12 +778,12 @@ public: return Op; } /// CreateRegMask - Creates a register mask operand referencing Mask. The - /// operand does not take ownership of the memory referenced by Mask, it must - /// remain valid for the lifetime of the operand. + /// operand does not take ownership of the memory referenced by Mask, it + /// must remain valid for the lifetime of the operand. /// - /// A RegMask operand represents a set of non-clobbered physical registers on - /// an instruction that clobbers many registers, typically a call. The bit - /// mask has a bit set for each physreg that is preserved by this + /// A RegMask operand represents a set of non-clobbered physical registers + /// on an instruction that clobbers many registers, typically a call. The + /// bit mask has a bit set for each physreg that is preserved by this /// instruction, as described in the documentation for /// TargetRegisterInfo::getCallPreservedMask(). /// @@ -769,30 +836,63 @@ public: friend class MachineInstr; friend class MachineRegisterInfo; + private: + // If this operand is currently a register operand, and if this is in a + // function, deregister the operand from the register's use/def list. void removeRegFromUses(); + /// Artificial kinds for DenseMap usage. + enum : unsigned char { + MO_Empty = MO_Last + 1, + MO_Tombstone, + }; + + friend struct DenseMapInfo<MachineOperand>; + //===--------------------------------------------------------------------===// // Methods for handling register use/def lists. //===--------------------------------------------------------------------===// - /// isOnRegUseList - Return true if this operand is on a register use/def list - /// or false if not. This can only be called for register operands that are - /// part of a machine instruction. + /// isOnRegUseList - Return true if this operand is on a register use/def + /// list or false if not. This can only be called for register operands + /// that are part of a machine instruction. bool isOnRegUseList() const { assert(isReg() && "Can only add reg operand to use lists"); return Contents.Reg.Prev != nullptr; } }; -inline raw_ostream &operator<<(raw_ostream &OS, const MachineOperand& MO) { - MO.print(OS, nullptr); +template <> struct DenseMapInfo<MachineOperand> { + static MachineOperand getEmptyKey() { + return MachineOperand(static_cast<MachineOperand::MachineOperandType>( + MachineOperand::MO_Empty)); + } + static MachineOperand getTombstoneKey() { + return MachineOperand(static_cast<MachineOperand::MachineOperandType>( + MachineOperand::MO_Tombstone)); + } + static unsigned getHashValue(const MachineOperand &MO) { + return hash_value(MO); + } + static bool isEqual(const MachineOperand &LHS, const MachineOperand &RHS) { + if (LHS.getType() == static_cast<MachineOperand::MachineOperandType>( + MachineOperand::MO_Empty) || + LHS.getType() == static_cast<MachineOperand::MachineOperandType>( + MachineOperand::MO_Tombstone)) + return LHS.getType() == RHS.getType(); + return LHS.isIdenticalTo(RHS); + } +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const MachineOperand &MO) { + MO.print(OS); return OS; } - // See friend declaration above. This additional declaration is required in - // order to compile LLVM with IBM xlC compiler. - hash_code hash_value(const MachineOperand &MO); -} // End llvm namespace +// See friend declaration above. This additional declaration is required in +// order to compile LLVM with IBM xlC compiler. +hash_code hash_value(const MachineOperand &MO); +} // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/contrib/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h index 6ad5de533d13..2fdefbed37ce 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h @@ -16,7 +16,7 @@ #ifndef LLVM_CODEGEN_MACHINEOPTIMIZATIONREMARKEMITTER_H #define LLVM_CODEGEN_MACHINEOPTIMIZATIONREMARKEMITTER_H -#include "llvm/Analysis/OptimizationDiagnosticInfo.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { @@ -33,7 +33,7 @@ public: const DiagnosticLocation &Loc, const MachineBasicBlock *MBB) : DiagnosticInfoOptimizationBase(Kind, DS_Remark, PassName, RemarkName, - *MBB->getParent()->getFunction(), Loc), + MBB->getParent()->getFunction(), Loc), MBB(MBB) {} /// MI-specific kinds of diagnostic Arguments. @@ -73,7 +73,9 @@ public: /// \see DiagnosticInfoOptimizationBase::isEnabled. bool isEnabled() const override { - return OptimizationRemark::isEnabled(getPassName()); + const Function &Fn = getFunction(); + LLVMContext &Ctx = Fn.getContext(); + return Ctx.getDiagHandlerPtr()->isPassedOptRemarkEnabled(getPassName()); } }; @@ -97,7 +99,9 @@ public: /// \see DiagnosticInfoOptimizationBase::isEnabled. bool isEnabled() const override { - return OptimizationRemarkMissed::isEnabled(getPassName()); + const Function &Fn = getFunction(); + LLVMContext &Ctx = Fn.getContext(); + return Ctx.getDiagHandlerPtr()->isMissedOptRemarkEnabled(getPassName()); } }; @@ -121,7 +125,9 @@ public: /// \see DiagnosticInfoOptimizationBase::isEnabled. bool isEnabled() const override { - return OptimizationRemarkAnalysis::isEnabled(getPassName()); + const Function &Fn = getFunction(); + LLVMContext &Ctx = Fn.getContext(); + return Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(getPassName()); } }; @@ -152,10 +158,25 @@ public: /// that are normally too noisy. In this mode, we can use the extra analysis /// (1) to filter trivial false positives or (2) to provide more context so /// that non-trivial false positives can be quickly detected by the user. - bool allowExtraAnalysis() const { - // For now, only allow this with -fsave-optimization-record since the -Rpass - // options are handled in the front-end. - return MF.getFunction()->getContext().getDiagnosticsOutputFile(); + bool allowExtraAnalysis(StringRef PassName) const { + return (MF.getFunction().getContext().getDiagnosticsOutputFile() || + MF.getFunction().getContext() + .getDiagHandlerPtr()->isAnyRemarkEnabled(PassName)); + } + + /// \brief Take a lambda that returns a remark which will be emitted. Second + /// argument is only used to restrict this to functions. + template <typename T> + void emit(T RemarkBuilder, decltype(RemarkBuilder()) * = nullptr) { + // Avoid building the remark unless we know there are at least *some* + // remarks enabled. We can't currently check whether remarks are requested + // for the calling pass since that requires actually building the remark. + + if (MF.getFunction().getContext().getDiagnosticsOutputFile() || + MF.getFunction().getContext().getDiagHandlerPtr()->isAnyRemarkEnabled()) { + auto R = RemarkBuilder(); + emit((DiagnosticInfoOptimizationBase &)R); + } } private: diff --git a/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h b/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h index d29d2d85cb0a..c6a41598ce32 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h +++ b/contrib/llvm/include/llvm/CodeGen/MachinePostDominators.h @@ -37,7 +37,7 @@ public: FunctionPass *createMachinePostDominatorTreePass(); - const std::vector<MachineBasicBlock *> &getRoots() const { + const SmallVectorImpl<MachineBasicBlock *> &getRoots() const { return DT->getRoots(); } diff --git a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h index 5ef0ac90e3c2..3be94f802170 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineRegisterInfo.h @@ -27,9 +27,9 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/MC/LaneBitmask.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include <cassert> #include <cstddef> #include <cstdint> @@ -84,14 +84,15 @@ private: /// all registers that were disabled are removed from the list. SmallVector<MCPhysReg, 16> UpdatedCSRs; - /// RegAllocHints - This vector records register allocation hints for virtual - /// registers. For each virtual register, it keeps a register and hint type - /// pair making up the allocation hint. Hint type is target specific except - /// for the value 0 which means the second value of the pair is the preferred - /// register for allocation. For example, if the hint is <0, 1024>, it means - /// the allocator should prefer the physical register allocated to the virtual - /// register of the hint. - IndexedMap<std::pair<unsigned, unsigned>, VirtReg2IndexFunctor> RegAllocHints; + /// RegAllocHints - This vector records register allocation hints for + /// virtual registers. For each virtual register, it keeps a pair of hint + /// type and hints vector making up the allocation hints. Only the first + /// hint may be target specific, and in that case this is reflected by the + /// first member of the pair being non-zero. If the hinted register is + /// virtual, it means the allocator should prefer the physical register + /// allocated to it if any. + IndexedMap<std::pair<unsigned, SmallVector<unsigned, 4>>, + VirtReg2IndexFunctor> RegAllocHints; /// PhysRegUseDefLists - This is an array of the head of the use/def list for /// physical registers. @@ -575,14 +576,16 @@ public: /// preserve conservative kill flag information. void clearKillFlags(unsigned Reg) const; -#ifndef NDEBUG void dumpUses(unsigned RegNo) const; -#endif /// Returns true if PhysReg is unallocatable and constant throughout the /// function. Writing to a constant register has no effect. bool isConstantPhysReg(unsigned PhysReg) const; + /// Returns true if either isConstantPhysReg or TRI->isCallerPreservedPhysReg + /// returns true. This is a utility member function. + bool isCallerPreservedOrConstPhysReg(unsigned PhysReg) const; + /// Get an iterator over the pressure sets affected by the given physical or /// virtual register. If RegUnit is physical, it must be a register unit (from /// MCRegUnitIterator). @@ -704,35 +707,61 @@ public: void clearVirtRegs(); /// setRegAllocationHint - Specify a register allocation hint for the - /// specified virtual register. + /// specified virtual register. This is typically used by target, and in case + /// of an earlier hint it will be overwritten. void setRegAllocationHint(unsigned VReg, unsigned Type, unsigned PrefReg) { assert(TargetRegisterInfo::isVirtualRegister(VReg)); RegAllocHints[VReg].first = Type; - RegAllocHints[VReg].second = PrefReg; + RegAllocHints[VReg].second.clear(); + RegAllocHints[VReg].second.push_back(PrefReg); } - /// Specify the preferred register allocation hint for the specified virtual - /// register. + /// addRegAllocationHint - Add a register allocation hint to the hints + /// vector for VReg. + void addRegAllocationHint(unsigned VReg, unsigned PrefReg) { + assert(TargetRegisterInfo::isVirtualRegister(VReg)); + RegAllocHints[VReg].second.push_back(PrefReg); + } + + /// Specify the preferred (target independent) register allocation hint for + /// the specified virtual register. void setSimpleHint(unsigned VReg, unsigned PrefReg) { setRegAllocationHint(VReg, /*Type=*/0, PrefReg); } + void clearSimpleHint(unsigned VReg) { + assert (RegAllocHints[VReg].first == 0 && + "Expected to clear a non-target hint!"); + RegAllocHints[VReg].second.clear(); + } + /// getRegAllocationHint - Return the register allocation hint for the - /// specified virtual register. + /// specified virtual register. If there are many hints, this returns the + /// one with the greatest weight. std::pair<unsigned, unsigned> getRegAllocationHint(unsigned VReg) const { assert(TargetRegisterInfo::isVirtualRegister(VReg)); - return RegAllocHints[VReg]; + unsigned BestHint = (RegAllocHints[VReg].second.size() ? + RegAllocHints[VReg].second[0] : 0); + return std::pair<unsigned, unsigned>(RegAllocHints[VReg].first, BestHint); } - /// getSimpleHint - Return the preferred register allocation hint, or 0 if a - /// standard simple hint (Type == 0) is not set. + /// getSimpleHint - same as getRegAllocationHint except it will only return + /// a target independent hint. unsigned getSimpleHint(unsigned VReg) const { assert(TargetRegisterInfo::isVirtualRegister(VReg)); std::pair<unsigned, unsigned> Hint = getRegAllocationHint(VReg); return Hint.first ? 0 : Hint.second; } + /// getRegAllocationHints - Return a reference to the vector of all + /// register allocation hints for VReg. + const std::pair<unsigned, SmallVector<unsigned, 4>> + &getRegAllocationHints(unsigned VReg) const { + assert(TargetRegisterInfo::isVirtualRegister(VReg)); + return RegAllocHints[VReg]; + } + /// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the /// specified register as undefined which causes the DBG_VALUE to be /// deleted during LiveDebugVariables analysis. @@ -844,6 +873,10 @@ public: livein_iterator livein_end() const { return LiveIns.end(); } bool livein_empty() const { return LiveIns.empty(); } + ArrayRef<std::pair<unsigned, unsigned>> liveins() const { + return LiveIns; + } + bool isLiveIn(unsigned Reg) const; /// getLiveInPhysReg - If VReg is a live-in virtual register, return the diff --git a/contrib/llvm/include/llvm/CodeGen/MachineSSAUpdater.h b/contrib/llvm/include/llvm/CodeGen/MachineSSAUpdater.h index 50a7d90bf25b..b5ea2080444d 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineSSAUpdater.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineSSAUpdater.h @@ -1,4 +1,4 @@ -//===-- MachineSSAUpdater.h - Unstructured SSA Update Tool ------*- C++ -*-===// +//===- MachineSSAUpdater.h - Unstructured SSA Update Tool -------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,18 +14,17 @@ #ifndef LLVM_CODEGEN_MACHINESSAUPDATER_H #define LLVM_CODEGEN_MACHINESSAUPDATER_H -#include "llvm/Support/Compiler.h" - namespace llvm { - class MachineBasicBlock; - class MachineFunction; - class MachineInstr; - class MachineOperand; - class MachineRegisterInfo; - class TargetInstrInfo; - class TargetRegisterClass; - template<typename T> class SmallVectorImpl; - template<typename T> class SSAUpdaterTraits; + +class MachineBasicBlock; +class MachineFunction; +class MachineInstr; +class MachineOperand; +class MachineRegisterInfo; +class TargetInstrInfo; +class TargetRegisterClass; +template<typename T> class SmallVectorImpl; +template<typename T> class SSAUpdaterTraits; /// MachineSSAUpdater - This class updates SSA form for a set of virtual /// registers defined in multiple blocks. This is used when code duplication @@ -38,7 +37,7 @@ private: /// AvailableVals - This keeps track of which value to use on a per-block /// basis. When we insert PHI nodes, we keep track of them here. //typedef DenseMap<MachineBasicBlock*, unsigned > AvailableValsTy; - void *AV; + void *AV = nullptr; /// VR - Current virtual register whose uses are being updated. unsigned VR; @@ -52,11 +51,14 @@ private: const TargetInstrInfo *TII; MachineRegisterInfo *MRI; + public: /// MachineSSAUpdater constructor. If InsertedPHIs is specified, it will be /// filled in with all PHI Nodes created by rewriting. explicit MachineSSAUpdater(MachineFunction &MF, SmallVectorImpl<MachineInstr*> *InsertedPHIs = nullptr); + MachineSSAUpdater(const MachineSSAUpdater &) = delete; + MachineSSAUpdater &operator=(const MachineSSAUpdater &) = delete; ~MachineSSAUpdater(); /// Initialize - Reset this object to get ready for a new set of SSA @@ -93,7 +95,6 @@ public: /// their respective blocks. However, the use of X happens in the *middle* of /// a block. Because of this, we need to insert a new PHI node in SomeBB to /// merge the appropriate values, and this value isn't live out of the block. - /// unsigned GetValueInMiddleOfBlock(MachineBasicBlock *BB); /// RewriteUse - Rewrite a use of the symbolic value. This handles PHI nodes, @@ -105,11 +106,8 @@ public: private: unsigned GetValueAtEndOfBlockInternal(MachineBasicBlock *BB); - - void operator=(const MachineSSAUpdater&) = delete; - MachineSSAUpdater(const MachineSSAUpdater&) = delete; }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_MACHINESSAUPDATER_H diff --git a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h index 8590b7a348cf..e327881de13a 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -26,7 +26,7 @@ // The default scheduler, ScheduleDAGMILive, builds the DAG and drives list // scheduling while updating the instruction stream, register pressure, and live // intervals. Most targets don't need to override the DAG builder and list -// schedulier, but subtargets that require custom scheduling heuristics may +// scheduler, but subtargets that require custom scheduling heuristics may // plugin an alternate MachineSchedStrategy. The strategy is responsible for // selecting the highest priority node from the list: // @@ -214,9 +214,20 @@ public: /// This has to be enabled in combination with shouldTrackPressure(). virtual bool shouldTrackLaneMasks() const { return false; } + // If this method returns true, handling of the scheduling regions + // themselves (in case of a scheduling boundary in MBB) will be done + // beginning with the topmost region of MBB. + virtual bool doMBBSchedRegionsTopDown() const { return false; } + /// Initialize the strategy after building the DAG for a new region. virtual void initialize(ScheduleDAGMI *DAG) = 0; + /// Tell the strategy that MBB is about to be processed. + virtual void enterMBB(MachineBasicBlock *MBB) {}; + + /// Tell the strategy that current MBB is done. + virtual void leaveMBB() {}; + /// Notify this strategy that all roots have been released (including those /// that depend on EntrySU or ExitSU). virtual void registerRoots() {} @@ -284,6 +295,13 @@ public: // Provide a vtable anchor ~ScheduleDAGMI() override; + /// If this method returns true, handling of the scheduling regions + /// themselves (in case of a scheduling boundary in MBB) will be done + /// beginning with the topmost region of MBB. + bool doMBBSchedRegionsTopDown() const override { + return SchedImpl->doMBBSchedRegionsTopDown(); + } + // Returns LiveIntervals instance for use in DAG mutators and such. LiveIntervals *getLIS() const { return LIS; } @@ -326,6 +344,9 @@ public: /// reorderable instructions. void schedule() override; + void startBlock(MachineBasicBlock *bb) override; + void finishBlock() override; + /// Change the position of an instruction within the basic block and update /// live ranges and region boundary iterators. void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos); @@ -755,9 +776,7 @@ public: /// available instruction, or NULL if there are multiple candidates. SUnit *pickOnlyChoice(); -#ifndef NDEBUG void dumpScheduledState() const; -#endif }; /// Base class for GenericScheduler. This class maintains information about diff --git a/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h b/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h index 284f8c197607..9d8db393ca92 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineTraceMetrics.h @@ -47,17 +47,18 @@ #ifndef LLVM_CODEGEN_MACHINETRACEMETRICS_H #define LLVM_CODEGEN_MACHINETRACEMETRICS_H +#include "llvm/ADT/SparseSet.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/TargetSchedule.h" namespace llvm { class AnalysisUsage; -class MachineBasicBlock; class MachineFunction; class MachineInstr; class MachineLoop; @@ -68,6 +69,22 @@ class raw_ostream; class TargetInstrInfo; class TargetRegisterInfo; +// Keep track of physreg data dependencies by recording each live register unit. +// Associate each regunit with an instruction operand. Depending on the +// direction instructions are scanned, it could be the operand that defined the +// regunit, or the highest operand to read the regunit. +struct LiveRegUnit { + unsigned RegUnit; + unsigned Cycle = 0; + const MachineInstr *MI = nullptr; + unsigned Op = 0; + + unsigned getSparseSetIndex() const { return RegUnit; } + + LiveRegUnit(unsigned RU) : RegUnit(RU) {} +}; + + class MachineTraceMetrics : public MachineFunctionPass { const MachineFunction *MF = nullptr; const TargetInstrInfo *TII = nullptr; @@ -343,6 +360,18 @@ public: /// Get the trace that passes through MBB. /// The trace is computed on demand. Trace getTrace(const MachineBasicBlock *MBB); + + /// Updates the depth of an machine instruction, given RegUnits. + void updateDepth(TraceBlockInfo &TBI, const MachineInstr&, + SparseSet<LiveRegUnit> &RegUnits); + void updateDepth(const MachineBasicBlock *, const MachineInstr&, + SparseSet<LiveRegUnit> &RegUnits); + + /// Updates the depth of the instructions from Start to End. + void updateDepths(MachineBasicBlock::iterator Start, + MachineBasicBlock::iterator End, + SparseSet<LiveRegUnit> &RegUnits); + }; /// Strategies for selecting traces. diff --git a/contrib/llvm/include/llvm/CodeGen/MachineValueType.h b/contrib/llvm/include/llvm/CodeGen/MachineValueType.h index 0bdb38bfcbec..b452684757f6 100644 --- a/contrib/llvm/include/llvm/CodeGen/MachineValueType.h +++ b/contrib/llvm/include/llvm/CodeGen/MachineValueType.h @@ -64,80 +64,81 @@ namespace llvm { v16i1 = 18, // 16 x i1 v32i1 = 19, // 32 x i1 v64i1 = 20, // 64 x i1 - v512i1 = 21, // 512 x i1 - v1024i1 = 22, // 1024 x i1 - - v1i8 = 23, // 1 x i8 - v2i8 = 24, // 2 x i8 - v4i8 = 25, // 4 x i8 - v8i8 = 26, // 8 x i8 - v16i8 = 27, // 16 x i8 - v32i8 = 28, // 32 x i8 - v64i8 = 29, // 64 x i8 - v128i8 = 30, //128 x i8 - v256i8 = 31, //256 x i8 - - v1i16 = 32, // 1 x i16 - v2i16 = 33, // 2 x i16 - v4i16 = 34, // 4 x i16 - v8i16 = 35, // 8 x i16 - v16i16 = 36, // 16 x i16 - v32i16 = 37, // 32 x i16 - v64i16 = 38, // 64 x i16 - v128i16 = 39, //128 x i16 - - v1i32 = 40, // 1 x i32 - v2i32 = 41, // 2 x i32 - v4i32 = 42, // 4 x i32 - v8i32 = 43, // 8 x i32 - v16i32 = 44, // 16 x i32 - v32i32 = 45, // 32 x i32 - v64i32 = 46, // 64 x i32 - - v1i64 = 47, // 1 x i64 - v2i64 = 48, // 2 x i64 - v4i64 = 49, // 4 x i64 - v8i64 = 50, // 8 x i64 - v16i64 = 51, // 16 x i64 - v32i64 = 52, // 32 x i64 - - v1i128 = 53, // 1 x i128 + v128i1 = 21, // 128 x i1 + v512i1 = 22, // 512 x i1 + v1024i1 = 23, // 1024 x i1 + + v1i8 = 24, // 1 x i8 + v2i8 = 25, // 2 x i8 + v4i8 = 26, // 4 x i8 + v8i8 = 27, // 8 x i8 + v16i8 = 28, // 16 x i8 + v32i8 = 29, // 32 x i8 + v64i8 = 30, // 64 x i8 + v128i8 = 31, //128 x i8 + v256i8 = 32, //256 x i8 + + v1i16 = 33, // 1 x i16 + v2i16 = 34, // 2 x i16 + v4i16 = 35, // 4 x i16 + v8i16 = 36, // 8 x i16 + v16i16 = 37, // 16 x i16 + v32i16 = 38, // 32 x i16 + v64i16 = 39, // 64 x i16 + v128i16 = 40, //128 x i16 + + v1i32 = 41, // 1 x i32 + v2i32 = 42, // 2 x i32 + v4i32 = 43, // 4 x i32 + v8i32 = 44, // 8 x i32 + v16i32 = 45, // 16 x i32 + v32i32 = 46, // 32 x i32 + v64i32 = 47, // 64 x i32 + + v1i64 = 48, // 1 x i64 + v2i64 = 49, // 2 x i64 + v4i64 = 50, // 4 x i64 + v8i64 = 51, // 8 x i64 + v16i64 = 52, // 16 x i64 + v32i64 = 53, // 32 x i64 + + v1i128 = 54, // 1 x i128 // Scalable integer types - nxv1i1 = 54, // n x 1 x i1 - nxv2i1 = 55, // n x 2 x i1 - nxv4i1 = 56, // n x 4 x i1 - nxv8i1 = 57, // n x 8 x i1 - nxv16i1 = 58, // n x 16 x i1 - nxv32i1 = 59, // n x 32 x i1 - - nxv1i8 = 60, // n x 1 x i8 - nxv2i8 = 61, // n x 2 x i8 - nxv4i8 = 62, // n x 4 x i8 - nxv8i8 = 63, // n x 8 x i8 - nxv16i8 = 64, // n x 16 x i8 - nxv32i8 = 65, // n x 32 x i8 - - nxv1i16 = 66, // n x 1 x i16 - nxv2i16 = 67, // n x 2 x i16 - nxv4i16 = 68, // n x 4 x i16 - nxv8i16 = 69, // n x 8 x i16 - nxv16i16 = 70, // n x 16 x i16 - nxv32i16 = 71, // n x 32 x i16 - - nxv1i32 = 72, // n x 1 x i32 - nxv2i32 = 73, // n x 2 x i32 - nxv4i32 = 74, // n x 4 x i32 - nxv8i32 = 75, // n x 8 x i32 - nxv16i32 = 76, // n x 16 x i32 - nxv32i32 = 77, // n x 32 x i32 - - nxv1i64 = 78, // n x 1 x i64 - nxv2i64 = 79, // n x 2 x i64 - nxv4i64 = 80, // n x 4 x i64 - nxv8i64 = 81, // n x 8 x i64 - nxv16i64 = 82, // n x 16 x i64 - nxv32i64 = 83, // n x 32 x i64 + nxv1i1 = 55, // n x 1 x i1 + nxv2i1 = 56, // n x 2 x i1 + nxv4i1 = 57, // n x 4 x i1 + nxv8i1 = 58, // n x 8 x i1 + nxv16i1 = 59, // n x 16 x i1 + nxv32i1 = 60, // n x 32 x i1 + + nxv1i8 = 61, // n x 1 x i8 + nxv2i8 = 62, // n x 2 x i8 + nxv4i8 = 63, // n x 4 x i8 + nxv8i8 = 64, // n x 8 x i8 + nxv16i8 = 65, // n x 16 x i8 + nxv32i8 = 66, // n x 32 x i8 + + nxv1i16 = 67, // n x 1 x i16 + nxv2i16 = 68, // n x 2 x i16 + nxv4i16 = 69, // n x 4 x i16 + nxv8i16 = 70, // n x 8 x i16 + nxv16i16 = 71, // n x 16 x i16 + nxv32i16 = 72, // n x 32 x i16 + + nxv1i32 = 73, // n x 1 x i32 + nxv2i32 = 74, // n x 2 x i32 + nxv4i32 = 75, // n x 4 x i32 + nxv8i32 = 76, // n x 8 x i32 + nxv16i32 = 77, // n x 16 x i32 + nxv32i32 = 78, // n x 32 x i32 + + nxv1i64 = 79, // n x 1 x i64 + nxv2i64 = 80, // n x 2 x i64 + nxv4i64 = 81, // n x 4 x i64 + nxv8i64 = 82, // n x 8 x i64 + nxv16i64 = 83, // n x 16 x i64 + nxv32i64 = 84, // n x 32 x i64 FIRST_INTEGER_VECTOR_VALUETYPE = v1i1, LAST_INTEGER_VECTOR_VALUETYPE = nxv32i64, @@ -145,31 +146,31 @@ namespace llvm { FIRST_INTEGER_SCALABLE_VALUETYPE = nxv1i1, LAST_INTEGER_SCALABLE_VALUETYPE = nxv32i64, - v2f16 = 84, // 2 x f16 - v4f16 = 85, // 4 x f16 - v8f16 = 86, // 8 x f16 - v1f32 = 87, // 1 x f32 - v2f32 = 88, // 2 x f32 - v4f32 = 89, // 4 x f32 - v8f32 = 90, // 8 x f32 - v16f32 = 91, // 16 x f32 - v1f64 = 92, // 1 x f64 - v2f64 = 93, // 2 x f64 - v4f64 = 94, // 4 x f64 - v8f64 = 95, // 8 x f64 - - nxv2f16 = 96, // n x 2 x f16 - nxv4f16 = 97, // n x 4 x f16 - nxv8f16 = 98, // n x 8 x f16 - nxv1f32 = 99, // n x 1 x f32 - nxv2f32 = 100, // n x 2 x f32 - nxv4f32 = 101, // n x 4 x f32 - nxv8f32 = 102, // n x 8 x f32 - nxv16f32 = 103, // n x 16 x f32 - nxv1f64 = 104, // n x 1 x f64 - nxv2f64 = 105, // n x 2 x f64 - nxv4f64 = 106, // n x 4 x f64 - nxv8f64 = 107, // n x 8 x f64 + v2f16 = 85, // 2 x f16 + v4f16 = 86, // 4 x f16 + v8f16 = 87, // 8 x f16 + v1f32 = 88, // 1 x f32 + v2f32 = 89, // 2 x f32 + v4f32 = 90, // 4 x f32 + v8f32 = 91, // 8 x f32 + v16f32 = 92, // 16 x f32 + v1f64 = 93, // 1 x f64 + v2f64 = 94, // 2 x f64 + v4f64 = 95, // 4 x f64 + v8f64 = 96, // 8 x f64 + + nxv2f16 = 97, // n x 2 x f16 + nxv4f16 = 98, // n x 4 x f16 + nxv8f16 = 99, // n x 8 x f16 + nxv1f32 = 100, // n x 1 x f32 + nxv2f32 = 101, // n x 2 x f32 + nxv4f32 = 102, // n x 4 x f32 + nxv8f32 = 103, // n x 8 x f32 + nxv16f32 = 104, // n x 16 x f32 + nxv1f64 = 105, // n x 1 x f64 + nxv2f64 = 106, // n x 2 x f64 + nxv4f64 = 107, // n x 4 x f64 + nxv8f64 = 108, // n x 8 x f64 FIRST_FP_VECTOR_VALUETYPE = v2f16, LAST_FP_VECTOR_VALUETYPE = nxv8f64, @@ -180,18 +181,18 @@ namespace llvm { FIRST_VECTOR_VALUETYPE = v1i1, LAST_VECTOR_VALUETYPE = nxv8f64, - x86mmx = 108, // This is an X86 MMX value + x86mmx = 109, // This is an X86 MMX value - Glue = 109, // This glues nodes together during pre-RA sched + Glue = 110, // This glues nodes together during pre-RA sched - isVoid = 110, // This has no value + isVoid = 111, // This has no value - Untyped = 111, // This value takes a register, but has + Untyped = 112, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. FIRST_VALUETYPE = 1, // This is always the beginning of the list. - LAST_VALUETYPE = 112, // This always remains at the end of the list. + LAST_VALUETYPE = 113, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors @@ -346,10 +347,11 @@ namespace llvm { /// Return true if this is a 128-bit vector type. bool is128BitVector() const { - return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 || - SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 || - SimpleTy == MVT::v1i128 || SimpleTy == MVT::v8f16 || - SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64); + return (SimpleTy == MVT::v128i1 || SimpleTy == MVT::v16i8 || + SimpleTy == MVT::v8i16 || SimpleTy == MVT::v4i32 || + SimpleTy == MVT::v2i64 || SimpleTy == MVT::v1i128 || + SimpleTy == MVT::v8f16 || SimpleTy == MVT::v4f32 || + SimpleTy == MVT::v2f64); } /// Return true if this is a 256-bit vector type. @@ -420,6 +422,7 @@ namespace llvm { case v16i1: case v32i1: case v64i1: + case v128i1: case v512i1: case v1024i1: case nxv1i1: @@ -517,6 +520,7 @@ namespace llvm { case v1024i1: return 1024; case v512i1: return 512; case v256i8: return 256; + case v128i1: case v128i8: case v128i16: return 128; case v64i1: @@ -690,6 +694,7 @@ namespace llvm { case f128: case ppcf128: case i128: + case v128i1: case v16i8: case v8i16: case v4i32: @@ -828,6 +833,7 @@ namespace llvm { if (NumElements == 16) return MVT::v16i1; if (NumElements == 32) return MVT::v32i1; if (NumElements == 64) return MVT::v64i1; + if (NumElements == 128) return MVT::v128i1; if (NumElements == 512) return MVT::v512i1; if (NumElements == 1024) return MVT::v1024i1; break; diff --git a/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h b/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h index 8d5d2374679d..6a247277fdfa 100644 --- a/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h +++ b/contrib/llvm/include/llvm/CodeGen/PBQP/Solution.h @@ -29,11 +29,6 @@ namespace PBQP { using SelectionsMap = std::map<GraphBase::NodeId, unsigned>; SelectionsMap selections; - unsigned r0Reductions = 0; - unsigned r1Reductions = 0; - unsigned r2Reductions = 0; - unsigned rNReductions = 0; - public: /// \brief Initialise an empty solution. Solution() = default; diff --git a/contrib/llvm/include/llvm/CodeGen/Passes.h b/contrib/llvm/include/llvm/CodeGen/Passes.h index 96cfce5b84df..4370d116e08c 100644 --- a/contrib/llvm/include/llvm/CodeGen/Passes.h +++ b/contrib/llvm/include/llvm/CodeGen/Passes.h @@ -43,9 +43,6 @@ namespace llvm { /// the entry block. FunctionPass *createUnreachableBlockEliminationPass(); - /// Insert mcount-like function calls. - FunctionPass *createCountingFunctionInserterPass(); - /// MachineFunctionPrinter pass - This pass prints out the machine function to /// the given stream as a debugging tool. MachineFunctionPass * @@ -409,17 +406,17 @@ namespace llvm { /// This pass frees the memory occupied by the MachineFunction. FunctionPass *createFreeMachineFunctionPass(); - /// This pass combine basic blocks guarded by the same branch. - extern char &BranchCoalescingID; - /// This pass performs outlining on machine instructions directly before /// printing assembly. - ModulePass *createMachineOutlinerPass(); + ModulePass *createMachineOutlinerPass(bool OutlineFromLinkOnceODRs = false); /// This pass expands the experimental reduction intrinsics into sequences of /// shuffles. FunctionPass *createExpandReductionsPass(); + // This pass expands memcmp() to load/stores. + FunctionPass *createExpandMemCmpPass(); + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h b/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h index 765ca085244a..7a007eb8bcea 100644 --- a/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h @@ -1,4 +1,4 @@ -//===--- PreISelIntrinsicLowering.h - Pre-ISel intrinsic lowering pass ----===// +//===- PreISelIntrinsicLowering.h - Pre-ISel intrinsic lowering pass ------===// // // The LLVM Compiler Infrastructure // @@ -17,10 +17,13 @@ namespace llvm { +class Module; + struct PreISelIntrinsicLoweringPass : PassInfoMixin<PreISelIntrinsicLoweringPass> { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_CODEGEN_PREISELINTRINSICLOWERING_H diff --git a/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h b/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h index f5aedb07e4d2..bdf0bb731540 100644 --- a/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h +++ b/contrib/llvm/include/llvm/CodeGen/PseudoSourceValue.h @@ -25,6 +25,7 @@ namespace llvm { class MachineFrameInfo; class MachineMemOperand; class raw_ostream; +class TargetInstrInfo; raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO); class PseudoSourceValue; @@ -48,6 +49,7 @@ public: private: PSVKind Kind; + unsigned AddressSpace; friend raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV); @@ -58,7 +60,7 @@ private: virtual void printCustom(raw_ostream &O) const; public: - explicit PseudoSourceValue(PSVKind Kind); + explicit PseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); virtual ~PseudoSourceValue(); @@ -68,6 +70,9 @@ public: bool isGOT() const { return Kind == GOT; } bool isConstantPool() const { return Kind == ConstantPool; } bool isJumpTable() const { return Kind == JumpTable; } + + unsigned getAddressSpace() const { return AddressSpace; } + unsigned getTargetCustom() const { return (Kind >= TargetCustom) ? ((Kind+1) - TargetCustom) : 0; } @@ -91,8 +96,8 @@ class FixedStackPseudoSourceValue : public PseudoSourceValue { const int FI; public: - explicit FixedStackPseudoSourceValue(int FI) - : PseudoSourceValue(FixedStack), FI(FI) {} + explicit FixedStackPseudoSourceValue(int FI, const TargetInstrInfo &TII) + : PseudoSourceValue(FixedStack, TII), FI(FI) {} static bool classof(const PseudoSourceValue *V) { return V->kind() == FixedStack; @@ -111,7 +116,7 @@ public: class CallEntryPseudoSourceValue : public PseudoSourceValue { protected: - CallEntryPseudoSourceValue(PSVKind Kind); + CallEntryPseudoSourceValue(PSVKind Kind, const TargetInstrInfo &TII); public: bool isConstant(const MachineFrameInfo *) const override; @@ -124,7 +129,8 @@ class GlobalValuePseudoSourceValue : public CallEntryPseudoSourceValue { const GlobalValue *GV; public: - GlobalValuePseudoSourceValue(const GlobalValue *GV); + GlobalValuePseudoSourceValue(const GlobalValue *GV, + const TargetInstrInfo &TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == GlobalValueCallEntry; @@ -138,7 +144,7 @@ class ExternalSymbolPseudoSourceValue : public CallEntryPseudoSourceValue { const char *ES; public: - ExternalSymbolPseudoSourceValue(const char *ES); + ExternalSymbolPseudoSourceValue(const char *ES, const TargetInstrInfo &TII); static bool classof(const PseudoSourceValue *V) { return V->kind() == ExternalSymbolCallEntry; @@ -149,6 +155,7 @@ public: /// Manages creation of pseudo source values. class PseudoSourceValueManager { + const TargetInstrInfo &TII; const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; std::map<int, std::unique_ptr<FixedStackPseudoSourceValue>> FSValues; StringMap<std::unique_ptr<const ExternalSymbolPseudoSourceValue>> @@ -158,7 +165,7 @@ class PseudoSourceValueManager { GlobalCallEntries; public: - PseudoSourceValueManager(); + PseudoSourceValueManager(const TargetInstrInfo &TII); /// Return a pseudo source value referencing the area below the stack frame of /// a function, e.g., the argument space. diff --git a/contrib/llvm/include/llvm/CodeGen/RegisterClassInfo.h b/contrib/llvm/include/llvm/CodeGen/RegisterClassInfo.h index 355c9f9b2f1e..97113c575815 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegisterClassInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/RegisterClassInfo.h @@ -20,8 +20,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <cstdint> #include <memory> diff --git a/contrib/llvm/include/llvm/CodeGen/RegisterPressure.h b/contrib/llvm/include/llvm/CodeGen/RegisterPressure.h index e997aaf269e3..2b14b78d621d 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegisterPressure.h +++ b/contrib/llvm/include/llvm/CodeGen/RegisterPressure.h @@ -20,8 +20,8 @@ #include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/LaneBitmask.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <cstddef> #include <cstdint> diff --git a/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h b/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h index 0a04bc6a89f4..eabadd8d784a 100644 --- a/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/RegisterUsageInfo.h @@ -20,6 +20,7 @@ #define LLVM_CODEGEN_PHYSICALREGISTERUSAGEINFO_H #include "llvm/ADT/DenseMap.h" +#include "llvm/IR/Instructions.h" #include "llvm/Pass.h" #include <cstdint> #include <vector> diff --git a/contrib/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h b/contrib/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h index 9c8f5f487d38..03166ccdfe38 100644 --- a/contrib/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h +++ b/contrib/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h @@ -20,15 +20,15 @@ #include "llvm/CodeGen/DFAPacketizer.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" namespace llvm { class ResourcePriorityQueue; /// Sorting functions for the Available queue. - struct resource_sort : public std::binary_function<SUnit*, SUnit*, bool> { + struct resource_sort { ResourcePriorityQueue *PQ; explicit resource_sort(ResourcePriorityQueue *pq) : PQ(pq) {} diff --git a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.def b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.def new file mode 100644 index 000000000000..e042ae982e86 --- /dev/null +++ b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.def @@ -0,0 +1,492 @@ +//===-- llvm/RuntimeLibcalls.def - File that describes libcalls -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines all of the runtime library calls the backend can emit. +// The various long double types cannot be merged, because 80-bit library +// functions use "xf" and 128-bit use "tf". +// +// When adding PPCF128 functions here, note that their names generally need +// to be overridden for Darwin with the xxx$LDBL128 form. See +// PPCISelLowering.cpp. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +// Provide definitions of macros so that users of this file do not have to +// define everything to use it... + +// Declare the enumerator for each libcall, along with its default name. Some +// libcalls have different names on particular OSes or architectures. These +// are set in InitLibcallNames() in TargetLoweringBase.cpp and/or by targets +// using TargetLoweringBase::setLibcallName() +#ifndef HANDLE_LIBCALL +#error "HANDLE_LIBCALL must be defined" +#endif + +// Integer +HANDLE_LIBCALL(SHL_I16, "__ashlhi3") +HANDLE_LIBCALL(SHL_I32, "__ashlsi3") +HANDLE_LIBCALL(SHL_I64, "__ashldi3") +HANDLE_LIBCALL(SHL_I128, "__ashlti3") +HANDLE_LIBCALL(SRL_I16, "__lshrhi3") +HANDLE_LIBCALL(SRL_I32, "__lshrsi3") +HANDLE_LIBCALL(SRL_I64, "__lshrdi3") +HANDLE_LIBCALL(SRL_I128, "__lshrti3") +HANDLE_LIBCALL(SRA_I16, "__ashrhi3") +HANDLE_LIBCALL(SRA_I32, "__ashrsi3") +HANDLE_LIBCALL(SRA_I64, "__ashrdi3") +HANDLE_LIBCALL(SRA_I128, "__ashrti3") +HANDLE_LIBCALL(MUL_I8, "__mulqi3") +HANDLE_LIBCALL(MUL_I16, "__mulhi3") +HANDLE_LIBCALL(MUL_I32, "__mulsi3") +HANDLE_LIBCALL(MUL_I64, "__muldi3") +HANDLE_LIBCALL(MUL_I128, "__multi3") +HANDLE_LIBCALL(MULO_I32, "__mulosi4") +HANDLE_LIBCALL(MULO_I64, "__mulodi4") +HANDLE_LIBCALL(MULO_I128, "__muloti4") +HANDLE_LIBCALL(SDIV_I8, "__divqi3") +HANDLE_LIBCALL(SDIV_I16, "__divhi3") +HANDLE_LIBCALL(SDIV_I32, "__divsi3") +HANDLE_LIBCALL(SDIV_I64, "__divdi3") +HANDLE_LIBCALL(SDIV_I128, "__divti3") +HANDLE_LIBCALL(UDIV_I8, "__udivqi3") +HANDLE_LIBCALL(UDIV_I16, "__udivhi3") +HANDLE_LIBCALL(UDIV_I32, "__udivsi3") +HANDLE_LIBCALL(UDIV_I64, "__udivdi3") +HANDLE_LIBCALL(UDIV_I128, "__udivti3") +HANDLE_LIBCALL(SREM_I8, "__modqi3") +HANDLE_LIBCALL(SREM_I16, "__modhi3") +HANDLE_LIBCALL(SREM_I32, "__modsi3") +HANDLE_LIBCALL(SREM_I64, "__moddi3") +HANDLE_LIBCALL(SREM_I128, "__modti3") +HANDLE_LIBCALL(UREM_I8, "__umodqi3") +HANDLE_LIBCALL(UREM_I16, "__umodhi3") +HANDLE_LIBCALL(UREM_I32, "__umodsi3") +HANDLE_LIBCALL(UREM_I64, "__umoddi3") +HANDLE_LIBCALL(UREM_I128, "__umodti3") +HANDLE_LIBCALL(SDIVREM_I8, nullptr) +HANDLE_LIBCALL(SDIVREM_I16, nullptr) +HANDLE_LIBCALL(SDIVREM_I32, nullptr) +HANDLE_LIBCALL(SDIVREM_I64, nullptr) +HANDLE_LIBCALL(SDIVREM_I128, nullptr) +HANDLE_LIBCALL(UDIVREM_I8, nullptr) +HANDLE_LIBCALL(UDIVREM_I16, nullptr) +HANDLE_LIBCALL(UDIVREM_I32, nullptr) +HANDLE_LIBCALL(UDIVREM_I64, nullptr) +HANDLE_LIBCALL(UDIVREM_I128, nullptr) +HANDLE_LIBCALL(NEG_I32, "__negsi2") +HANDLE_LIBCALL(NEG_I64, "__negdi2") + +// Floating-point +HANDLE_LIBCALL(ADD_F32, "__addsf3") +HANDLE_LIBCALL(ADD_F64, "__adddf3") +HANDLE_LIBCALL(ADD_F80, "__addxf3") +HANDLE_LIBCALL(ADD_F128, "__addtf3") +HANDLE_LIBCALL(ADD_PPCF128, "__gcc_qadd") +HANDLE_LIBCALL(SUB_F32, "__subsf3") +HANDLE_LIBCALL(SUB_F64, "__subdf3") +HANDLE_LIBCALL(SUB_F80, "__subxf3") +HANDLE_LIBCALL(SUB_F128, "__subtf3") +HANDLE_LIBCALL(SUB_PPCF128, "__gcc_qsub") +HANDLE_LIBCALL(MUL_F32, "__mulsf3") +HANDLE_LIBCALL(MUL_F64, "__muldf3") +HANDLE_LIBCALL(MUL_F80, "__mulxf3") +HANDLE_LIBCALL(MUL_F128, "__multf3") +HANDLE_LIBCALL(MUL_PPCF128, "__gcc_qmul") +HANDLE_LIBCALL(DIV_F32, "__divsf3") +HANDLE_LIBCALL(DIV_F64, "__divdf3") +HANDLE_LIBCALL(DIV_F80, "__divxf3") +HANDLE_LIBCALL(DIV_F128, "__divtf3") +HANDLE_LIBCALL(DIV_PPCF128, "__gcc_qdiv") +HANDLE_LIBCALL(REM_F32, "fmodf") +HANDLE_LIBCALL(REM_F64, "fmod") +HANDLE_LIBCALL(REM_F80, "fmodl") +HANDLE_LIBCALL(REM_F128, "fmodl") +HANDLE_LIBCALL(REM_PPCF128, "fmodl") +HANDLE_LIBCALL(FMA_F32, "fmaf") +HANDLE_LIBCALL(FMA_F64, "fma") +HANDLE_LIBCALL(FMA_F80, "fmal") +HANDLE_LIBCALL(FMA_F128, "fmal") +HANDLE_LIBCALL(FMA_PPCF128, "fmal") +HANDLE_LIBCALL(POWI_F32, "__powisf2") +HANDLE_LIBCALL(POWI_F64, "__powidf2") +HANDLE_LIBCALL(POWI_F80, "__powixf2") +HANDLE_LIBCALL(POWI_F128, "__powitf2") +HANDLE_LIBCALL(POWI_PPCF128, "__powitf2") +HANDLE_LIBCALL(SQRT_F32, "sqrtf") +HANDLE_LIBCALL(SQRT_F64, "sqrt") +HANDLE_LIBCALL(SQRT_F80, "sqrtl") +HANDLE_LIBCALL(SQRT_F128, "sqrtl") +HANDLE_LIBCALL(SQRT_PPCF128, "sqrtl") +HANDLE_LIBCALL(LOG_F32, "logf") +HANDLE_LIBCALL(LOG_F64, "log") +HANDLE_LIBCALL(LOG_F80, "logl") +HANDLE_LIBCALL(LOG_F128, "logl") +HANDLE_LIBCALL(LOG_PPCF128, "logl") +HANDLE_LIBCALL(LOG2_F32, "log2f") +HANDLE_LIBCALL(LOG2_F64, "log2") +HANDLE_LIBCALL(LOG2_F80, "log2l") +HANDLE_LIBCALL(LOG2_F128, "log2l") +HANDLE_LIBCALL(LOG2_PPCF128, "log2l") +HANDLE_LIBCALL(LOG10_F32, "log10f") +HANDLE_LIBCALL(LOG10_F64, "log10") +HANDLE_LIBCALL(LOG10_F80, "log10l") +HANDLE_LIBCALL(LOG10_F128, "log10l") +HANDLE_LIBCALL(LOG10_PPCF128, "log10l") +HANDLE_LIBCALL(EXP_F32, "expf") +HANDLE_LIBCALL(EXP_F64, "exp") +HANDLE_LIBCALL(EXP_F80, "expl") +HANDLE_LIBCALL(EXP_F128, "expl") +HANDLE_LIBCALL(EXP_PPCF128, "expl") +HANDLE_LIBCALL(EXP2_F32, "exp2f") +HANDLE_LIBCALL(EXP2_F64, "exp2") +HANDLE_LIBCALL(EXP2_F80, "exp2l") +HANDLE_LIBCALL(EXP2_F128, "exp2l") +HANDLE_LIBCALL(EXP2_PPCF128, "exp2l") +HANDLE_LIBCALL(SIN_F32, "sinf") +HANDLE_LIBCALL(SIN_F64, "sin") +HANDLE_LIBCALL(SIN_F80, "sinl") +HANDLE_LIBCALL(SIN_F128, "sinl") +HANDLE_LIBCALL(SIN_PPCF128, "sinl") +HANDLE_LIBCALL(COS_F32, "cosf") +HANDLE_LIBCALL(COS_F64, "cos") +HANDLE_LIBCALL(COS_F80, "cosl") +HANDLE_LIBCALL(COS_F128, "cosl") +HANDLE_LIBCALL(COS_PPCF128, "cosl") +HANDLE_LIBCALL(SINCOS_F32, nullptr) +HANDLE_LIBCALL(SINCOS_F64, nullptr) +HANDLE_LIBCALL(SINCOS_F80, nullptr) +HANDLE_LIBCALL(SINCOS_F128, nullptr) +HANDLE_LIBCALL(SINCOS_PPCF128, nullptr) +HANDLE_LIBCALL(POW_F32, "powf") +HANDLE_LIBCALL(POW_F64, "pow") +HANDLE_LIBCALL(POW_F80, "powl") +HANDLE_LIBCALL(POW_F128, "powl") +HANDLE_LIBCALL(POW_PPCF128, "powl") +HANDLE_LIBCALL(CEIL_F32, "ceilf") +HANDLE_LIBCALL(CEIL_F64, "ceil") +HANDLE_LIBCALL(CEIL_F80, "ceill") +HANDLE_LIBCALL(CEIL_F128, "ceill") +HANDLE_LIBCALL(CEIL_PPCF128, "ceill") +HANDLE_LIBCALL(TRUNC_F32, "truncf") +HANDLE_LIBCALL(TRUNC_F64, "trunc") +HANDLE_LIBCALL(TRUNC_F80, "truncl") +HANDLE_LIBCALL(TRUNC_F128, "truncl") +HANDLE_LIBCALL(TRUNC_PPCF128, "truncl") +HANDLE_LIBCALL(RINT_F32, "rintf") +HANDLE_LIBCALL(RINT_F64, "rint") +HANDLE_LIBCALL(RINT_F80, "rintl") +HANDLE_LIBCALL(RINT_F128, "rintl") +HANDLE_LIBCALL(RINT_PPCF128, "rintl") +HANDLE_LIBCALL(NEARBYINT_F32, "nearbyintf") +HANDLE_LIBCALL(NEARBYINT_F64, "nearbyint") +HANDLE_LIBCALL(NEARBYINT_F80, "nearbyintl") +HANDLE_LIBCALL(NEARBYINT_F128, "nearbyintl") +HANDLE_LIBCALL(NEARBYINT_PPCF128, "nearbyintl") +HANDLE_LIBCALL(ROUND_F32, "roundf") +HANDLE_LIBCALL(ROUND_F64, "round") +HANDLE_LIBCALL(ROUND_F80, "roundl") +HANDLE_LIBCALL(ROUND_F128, "roundl") +HANDLE_LIBCALL(ROUND_PPCF128, "roundl") +HANDLE_LIBCALL(FLOOR_F32, "floorf") +HANDLE_LIBCALL(FLOOR_F64, "floor") +HANDLE_LIBCALL(FLOOR_F80, "floorl") +HANDLE_LIBCALL(FLOOR_F128, "floorl") +HANDLE_LIBCALL(FLOOR_PPCF128, "floorl") +HANDLE_LIBCALL(COPYSIGN_F32, "copysignf") +HANDLE_LIBCALL(COPYSIGN_F64, "copysign") +HANDLE_LIBCALL(COPYSIGN_F80, "copysignl") +HANDLE_LIBCALL(COPYSIGN_F128, "copysignl") +HANDLE_LIBCALL(COPYSIGN_PPCF128, "copysignl") +HANDLE_LIBCALL(FMIN_F32, "fminf") +HANDLE_LIBCALL(FMIN_F64, "fmin") +HANDLE_LIBCALL(FMIN_F80, "fminl") +HANDLE_LIBCALL(FMIN_F128, "fminl") +HANDLE_LIBCALL(FMIN_PPCF128, "fminl") +HANDLE_LIBCALL(FMAX_F32, "fmaxf") +HANDLE_LIBCALL(FMAX_F64, "fmax") +HANDLE_LIBCALL(FMAX_F80, "fmaxl") +HANDLE_LIBCALL(FMAX_F128, "fmaxl") +HANDLE_LIBCALL(FMAX_PPCF128, "fmaxl") + +// Conversion +HANDLE_LIBCALL(FPEXT_F32_PPCF128, "__gcc_stoq") +HANDLE_LIBCALL(FPEXT_F64_PPCF128, "__gcc_dtoq") +HANDLE_LIBCALL(FPEXT_F64_F128, "__extenddftf2") +HANDLE_LIBCALL(FPEXT_F32_F128, "__extendsftf2") +HANDLE_LIBCALL(FPEXT_F32_F64, "__extendsfdf2") +HANDLE_LIBCALL(FPEXT_F16_F32, "__gnu_h2f_ieee") +HANDLE_LIBCALL(FPROUND_F32_F16, "__gnu_f2h_ieee") +HANDLE_LIBCALL(FPROUND_F64_F16, "__truncdfhf2") +HANDLE_LIBCALL(FPROUND_F80_F16, "__truncxfhf2") +HANDLE_LIBCALL(FPROUND_F128_F16, "__trunctfhf2") +HANDLE_LIBCALL(FPROUND_PPCF128_F16, "__trunctfhf2") +HANDLE_LIBCALL(FPROUND_F64_F32, "__truncdfsf2") +HANDLE_LIBCALL(FPROUND_F80_F32, "__truncxfsf2") +HANDLE_LIBCALL(FPROUND_F128_F32, "__trunctfsf2") +HANDLE_LIBCALL(FPROUND_PPCF128_F32, "__gcc_qtos") +HANDLE_LIBCALL(FPROUND_F80_F64, "__truncxfdf2") +HANDLE_LIBCALL(FPROUND_F128_F64, "__trunctfdf2") +HANDLE_LIBCALL(FPROUND_PPCF128_F64, "__gcc_qtod") +HANDLE_LIBCALL(FPTOSINT_F32_I32, "__fixsfsi") +HANDLE_LIBCALL(FPTOSINT_F32_I64, "__fixsfdi") +HANDLE_LIBCALL(FPTOSINT_F32_I128, "__fixsfti") +HANDLE_LIBCALL(FPTOSINT_F64_I32, "__fixdfsi") +HANDLE_LIBCALL(FPTOSINT_F64_I64, "__fixdfdi") +HANDLE_LIBCALL(FPTOSINT_F64_I128, "__fixdfti") +HANDLE_LIBCALL(FPTOSINT_F80_I32, "__fixxfsi") +HANDLE_LIBCALL(FPTOSINT_F80_I64, "__fixxfdi") +HANDLE_LIBCALL(FPTOSINT_F80_I128, "__fixxfti") +HANDLE_LIBCALL(FPTOSINT_F128_I32, "__fixtfsi") +HANDLE_LIBCALL(FPTOSINT_F128_I64, "__fixtfdi") +HANDLE_LIBCALL(FPTOSINT_F128_I128, "__fixtfti") +HANDLE_LIBCALL(FPTOSINT_PPCF128_I32, "__gcc_qtou") +HANDLE_LIBCALL(FPTOSINT_PPCF128_I64, "__fixtfdi") +HANDLE_LIBCALL(FPTOSINT_PPCF128_I128, "__fixtfti") +HANDLE_LIBCALL(FPTOUINT_F32_I32, "__fixunssfsi") +HANDLE_LIBCALL(FPTOUINT_F32_I64, "__fixunssfdi") +HANDLE_LIBCALL(FPTOUINT_F32_I128, "__fixunssfti") +HANDLE_LIBCALL(FPTOUINT_F64_I32, "__fixunsdfsi") +HANDLE_LIBCALL(FPTOUINT_F64_I64, "__fixunsdfdi") +HANDLE_LIBCALL(FPTOUINT_F64_I128, "__fixunsdfti") +HANDLE_LIBCALL(FPTOUINT_F80_I32, "__fixunsxfsi") +HANDLE_LIBCALL(FPTOUINT_F80_I64, "__fixunsxfdi") +HANDLE_LIBCALL(FPTOUINT_F80_I128, "__fixunsxfti") +HANDLE_LIBCALL(FPTOUINT_F128_I32, "__fixunstfsi") +HANDLE_LIBCALL(FPTOUINT_F128_I64, "__fixunstfdi") +HANDLE_LIBCALL(FPTOUINT_F128_I128, "__fixunstfti") +HANDLE_LIBCALL(FPTOUINT_PPCF128_I32, "__fixunstfsi") +HANDLE_LIBCALL(FPTOUINT_PPCF128_I64, "__fixunstfdi") +HANDLE_LIBCALL(FPTOUINT_PPCF128_I128, "__fixunstfti") +HANDLE_LIBCALL(SINTTOFP_I32_F32, "__floatsisf") +HANDLE_LIBCALL(SINTTOFP_I32_F64, "__floatsidf") +HANDLE_LIBCALL(SINTTOFP_I32_F80, "__floatsixf") +HANDLE_LIBCALL(SINTTOFP_I32_F128, "__floatsitf") +HANDLE_LIBCALL(SINTTOFP_I32_PPCF128, "__gcc_itoq") +HANDLE_LIBCALL(SINTTOFP_I64_F32, "__floatdisf") +HANDLE_LIBCALL(SINTTOFP_I64_F64, "__floatdidf") +HANDLE_LIBCALL(SINTTOFP_I64_F80, "__floatdixf") +HANDLE_LIBCALL(SINTTOFP_I64_F128, "__floatditf") +HANDLE_LIBCALL(SINTTOFP_I64_PPCF128, "__floatditf") +HANDLE_LIBCALL(SINTTOFP_I128_F32, "__floattisf") +HANDLE_LIBCALL(SINTTOFP_I128_F64, "__floattidf") +HANDLE_LIBCALL(SINTTOFP_I128_F80, "__floattixf") +HANDLE_LIBCALL(SINTTOFP_I128_F128, "__floattitf") +HANDLE_LIBCALL(SINTTOFP_I128_PPCF128, "__floattitf") +HANDLE_LIBCALL(UINTTOFP_I32_F32, "__floatunsisf") +HANDLE_LIBCALL(UINTTOFP_I32_F64, "__floatunsidf") +HANDLE_LIBCALL(UINTTOFP_I32_F80, "__floatunsixf") +HANDLE_LIBCALL(UINTTOFP_I32_F128, "__floatunsitf") +HANDLE_LIBCALL(UINTTOFP_I32_PPCF128, "__gcc_utoq") +HANDLE_LIBCALL(UINTTOFP_I64_F32, "__floatundisf") +HANDLE_LIBCALL(UINTTOFP_I64_F64, "__floatundidf") +HANDLE_LIBCALL(UINTTOFP_I64_F80, "__floatundixf") +HANDLE_LIBCALL(UINTTOFP_I64_F128, "__floatunditf") +HANDLE_LIBCALL(UINTTOFP_I64_PPCF128, "__floatunditf") +HANDLE_LIBCALL(UINTTOFP_I128_F32, "__floatuntisf") +HANDLE_LIBCALL(UINTTOFP_I128_F64, "__floatuntidf") +HANDLE_LIBCALL(UINTTOFP_I128_F80, "__floatuntixf") +HANDLE_LIBCALL(UINTTOFP_I128_F128, "__floatuntitf") +HANDLE_LIBCALL(UINTTOFP_I128_PPCF128, "__floatuntitf") + +// Comparison +HANDLE_LIBCALL(OEQ_F32, "__eqsf2") +HANDLE_LIBCALL(OEQ_F64, "__eqdf2") +HANDLE_LIBCALL(OEQ_F128, "__eqtf2") +HANDLE_LIBCALL(OEQ_PPCF128, "__gcc_qeq") +HANDLE_LIBCALL(UNE_F32, "__nesf2") +HANDLE_LIBCALL(UNE_F64, "__nedf2") +HANDLE_LIBCALL(UNE_F128, "__netf2") +HANDLE_LIBCALL(UNE_PPCF128, "__gcc_qne") +HANDLE_LIBCALL(OGE_F32, "__gesf2") +HANDLE_LIBCALL(OGE_F64, "__gedf2") +HANDLE_LIBCALL(OGE_F128, "__getf2") +HANDLE_LIBCALL(OGE_PPCF128, "__gcc_qge") +HANDLE_LIBCALL(OLT_F32, "__ltsf2") +HANDLE_LIBCALL(OLT_F64, "__ltdf2") +HANDLE_LIBCALL(OLT_F128, "__lttf2") +HANDLE_LIBCALL(OLT_PPCF128, "__gcc_qlt") +HANDLE_LIBCALL(OLE_F32, "__lesf2") +HANDLE_LIBCALL(OLE_F64, "__ledf2") +HANDLE_LIBCALL(OLE_F128, "__letf2") +HANDLE_LIBCALL(OLE_PPCF128, "__gcc_qle") +HANDLE_LIBCALL(OGT_F32, "__gtsf2") +HANDLE_LIBCALL(OGT_F64, "__gtdf2") +HANDLE_LIBCALL(OGT_F128, "__gttf2") +HANDLE_LIBCALL(OGT_PPCF128, "__gcc_qgt") +HANDLE_LIBCALL(UO_F32, "__unordsf2") +HANDLE_LIBCALL(UO_F64, "__unorddf2") +HANDLE_LIBCALL(UO_F128, "__unordtf2") +HANDLE_LIBCALL(UO_PPCF128, "__gcc_qunord") +HANDLE_LIBCALL(O_F32, "__unordsf2") +HANDLE_LIBCALL(O_F64, "__unorddf2") +HANDLE_LIBCALL(O_F128, "__unordtf2") +HANDLE_LIBCALL(O_PPCF128, "__gcc_qunord") + +// Memory +HANDLE_LIBCALL(MEMCPY, "memcpy") +HANDLE_LIBCALL(MEMMOVE, "memmove") +HANDLE_LIBCALL(MEMSET, "memset") + +// Element-wise unordered-atomic memory of different sizes +HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memcpy_element_unordered_atomic_1") +HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memcpy_element_unordered_atomic_2") +HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memcpy_element_unordered_atomic_4") +HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memcpy_element_unordered_atomic_8") +HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memcpy_element_unordered_atomic_16") +HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memmove_element_unordered_atomic_1") +HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memmove_element_unordered_atomic_2") +HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memmove_element_unordered_atomic_4") +HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memmove_element_unordered_atomic_8") +HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memmove_element_unordered_atomic_16") +HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memset_element_unordered_atomic_1") +HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memset_element_unordered_atomic_2") +HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memset_element_unordered_atomic_4") +HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memset_element_unordered_atomic_8") +HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memset_element_unordered_atomic_16") + +// Exception handling +HANDLE_LIBCALL(UNWIND_RESUME, "_Unwind_Resume") + +// Note: there are two sets of atomics libcalls; see +// <https://llvm.org/docs/Atomics.html> for more info on the +// difference between them. + +// Atomic '__sync_*' libcalls. +HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_1, "__sync_val_compare_and_swap_1") +HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_2, "__sync_val_compare_and_swap_2") +HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_4, "__sync_val_compare_and_swap_4") +HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_8, "__sync_val_compare_and_swap_8") +HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_16, "__sync_val_compare_and_swap_16") +HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_1, "__sync_lock_test_and_set_1") +HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_2, "__sync_lock_test_and_set_2") +HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_4, "__sync_lock_test_and_set_4") +HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_8, "__sync_lock_test_and_set_8") +HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_16, "__sync_lock_test_and_set_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_1, "__sync_fetch_and_add_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_2, "__sync_fetch_and_add_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_4, "__sync_fetch_and_add_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_8, "__sync_fetch_and_add_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_16, "__sync_fetch_and_add_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_1, "__sync_fetch_and_sub_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_2, "__sync_fetch_and_sub_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_4, "__sync_fetch_and_sub_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_8, "__sync_fetch_and_sub_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_16, "__sync_fetch_and_sub_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_AND_1, "__sync_fetch_and_and_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_AND_2, "__sync_fetch_and_and_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_AND_4, "__sync_fetch_and_and_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_AND_8, "__sync_fetch_and_and_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_AND_16, "__sync_fetch_and_and_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_OR_1, "__sync_fetch_and_or_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_OR_2, "__sync_fetch_and_or_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_OR_4, "__sync_fetch_and_or_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_OR_8, "__sync_fetch_and_or_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_OR_16, "__sync_fetch_and_or_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_1, "__sync_fetch_and_xor_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_2, "__sync_fetch_and_xor_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_4, "__sync_fetch_and_xor_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_8, "__sync_fetch_and_xor_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_16, "__sync_fetch_and_xor_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_1, "__sync_fetch_and_nand_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_2, "__sync_fetch_and_nand_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_4, "__sync_fetch_and_nand_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_8, "__sync_fetch_and_nand_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_16, "__sync_fetch_and_nand_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_1, "__sync_fetch_and_max_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_2, "__sync_fetch_and_max_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_4, "__sync_fetch_and_max_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_8, "__sync_fetch_and_max_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_16, "__sync_fetch_and_max_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_1, "__sync_fetch_and_umax_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_2, "__sync_fetch_and_umax_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_4, "__sync_fetch_and_umax_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_8, "__sync_fetch_and_umax_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_16, "__sync_fetch_and_umax_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_1, "__sync_fetch_and_min_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_2, "__sync_fetch_and_min_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_4, "__sync_fetch_and_min_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_8, "__sync_fetch_and_min_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_16, "__sync_fetch_and_min_16") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_1, "__sync_fetch_and_umin_1") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_2, "__sync_fetch_and_umin_2") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_4, "__sync_fetch_and_umin_4") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_8, "__sync_fetch_and_umin_8") +HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_16, "__sync_fetch_and_umin_16") + +// Atomic `__atomic_*' libcalls. +HANDLE_LIBCALL(ATOMIC_LOAD, "__atomic_load") +HANDLE_LIBCALL(ATOMIC_LOAD_1, "__atomic_load_1") +HANDLE_LIBCALL(ATOMIC_LOAD_2, "__atomic_load_2") +HANDLE_LIBCALL(ATOMIC_LOAD_4, "__atomic_load_4") +HANDLE_LIBCALL(ATOMIC_LOAD_8, "__atomic_load_8") +HANDLE_LIBCALL(ATOMIC_LOAD_16, "__atomic_load_16") + +HANDLE_LIBCALL(ATOMIC_STORE, "__atomic_store") +HANDLE_LIBCALL(ATOMIC_STORE_1, "__atomic_store_1") +HANDLE_LIBCALL(ATOMIC_STORE_2, "__atomic_store_2") +HANDLE_LIBCALL(ATOMIC_STORE_4, "__atomic_store_4") +HANDLE_LIBCALL(ATOMIC_STORE_8, "__atomic_store_8") +HANDLE_LIBCALL(ATOMIC_STORE_16, "__atomic_store_16") + +HANDLE_LIBCALL(ATOMIC_EXCHANGE, "__atomic_exchange") +HANDLE_LIBCALL(ATOMIC_EXCHANGE_1, "__atomic_exchange_1") +HANDLE_LIBCALL(ATOMIC_EXCHANGE_2, "__atomic_exchange_2") +HANDLE_LIBCALL(ATOMIC_EXCHANGE_4, "__atomic_exchange_4") +HANDLE_LIBCALL(ATOMIC_EXCHANGE_8, "__atomic_exchange_8") +HANDLE_LIBCALL(ATOMIC_EXCHANGE_16, "__atomic_exchange_16") + +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE, "__atomic_compare_exchange") +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_1, "__atomic_compare_exchange_1") +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_2, "__atomic_compare_exchange_2") +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_4, "__atomic_compare_exchange_4") +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_8, "__atomic_compare_exchange_8") +HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_16, "__atomic_compare_exchange_16") + +HANDLE_LIBCALL(ATOMIC_FETCH_ADD_1, "__atomic_fetch_add_1") +HANDLE_LIBCALL(ATOMIC_FETCH_ADD_2, "__atomic_fetch_add_2") +HANDLE_LIBCALL(ATOMIC_FETCH_ADD_4, "__atomic_fetch_add_4") +HANDLE_LIBCALL(ATOMIC_FETCH_ADD_8, "__atomic_fetch_add_8") +HANDLE_LIBCALL(ATOMIC_FETCH_ADD_16, "__atomic_fetch_add_16") +HANDLE_LIBCALL(ATOMIC_FETCH_SUB_1, "__atomic_fetch_sub_1") +HANDLE_LIBCALL(ATOMIC_FETCH_SUB_2, "__atomic_fetch_sub_2") +HANDLE_LIBCALL(ATOMIC_FETCH_SUB_4, "__atomic_fetch_sub_4") +HANDLE_LIBCALL(ATOMIC_FETCH_SUB_8, "__atomic_fetch_sub_8") +HANDLE_LIBCALL(ATOMIC_FETCH_SUB_16, "__atomic_fetch_sub_16") +HANDLE_LIBCALL(ATOMIC_FETCH_AND_1, "__atomic_fetch_and_1") +HANDLE_LIBCALL(ATOMIC_FETCH_AND_2, "__atomic_fetch_and_2") +HANDLE_LIBCALL(ATOMIC_FETCH_AND_4, "__atomic_fetch_and_4") +HANDLE_LIBCALL(ATOMIC_FETCH_AND_8, "__atomic_fetch_and_8") +HANDLE_LIBCALL(ATOMIC_FETCH_AND_16, "__atomic_fetch_and_16") +HANDLE_LIBCALL(ATOMIC_FETCH_OR_1, "__atomic_fetch_or_1") +HANDLE_LIBCALL(ATOMIC_FETCH_OR_2, "__atomic_fetch_or_2") +HANDLE_LIBCALL(ATOMIC_FETCH_OR_4, "__atomic_fetch_or_4") +HANDLE_LIBCALL(ATOMIC_FETCH_OR_8, "__atomic_fetch_or_8") +HANDLE_LIBCALL(ATOMIC_FETCH_OR_16, "__atomic_fetch_or_16") +HANDLE_LIBCALL(ATOMIC_FETCH_XOR_1, "__atomic_fetch_xor_1") +HANDLE_LIBCALL(ATOMIC_FETCH_XOR_2, "__atomic_fetch_xor_2") +HANDLE_LIBCALL(ATOMIC_FETCH_XOR_4, "__atomic_fetch_xor_4") +HANDLE_LIBCALL(ATOMIC_FETCH_XOR_8, "__atomic_fetch_xor_8") +HANDLE_LIBCALL(ATOMIC_FETCH_XOR_16, "__atomic_fetch_xor_16") +HANDLE_LIBCALL(ATOMIC_FETCH_NAND_1, "__atomic_fetch_nand_1") +HANDLE_LIBCALL(ATOMIC_FETCH_NAND_2, "__atomic_fetch_nand_2") +HANDLE_LIBCALL(ATOMIC_FETCH_NAND_4, "__atomic_fetch_nand_4") +HANDLE_LIBCALL(ATOMIC_FETCH_NAND_8, "__atomic_fetch_nand_8") +HANDLE_LIBCALL(ATOMIC_FETCH_NAND_16, "__atomic_fetch_nand_16") + +// Stack Protector Fail +HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail") + +// Deoptimization +HANDLE_LIBCALL(DEOPTIMIZE, "__llvm_deoptimize") + +HANDLE_LIBCALL(UNKNOWN_LIBCALL, nullptr) + +#undef HANDLE_LIBCALL diff --git a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h index 08151be11083..016bef1702c4 100644 --- a/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/contrib/llvm/include/llvm/CodeGen/RuntimeLibcalls.h @@ -28,471 +28,9 @@ namespace RTLIB { /// PPCISelLowering.cpp. /// enum Libcall { - // Integer - SHL_I16, - SHL_I32, - SHL_I64, - SHL_I128, - SRL_I16, - SRL_I32, - SRL_I64, - SRL_I128, - SRA_I16, - SRA_I32, - SRA_I64, - SRA_I128, - MUL_I8, - MUL_I16, - MUL_I32, - MUL_I64, - MUL_I128, - MULO_I32, - MULO_I64, - MULO_I128, - SDIV_I8, - SDIV_I16, - SDIV_I32, - SDIV_I64, - SDIV_I128, - UDIV_I8, - UDIV_I16, - UDIV_I32, - UDIV_I64, - UDIV_I128, - SREM_I8, - SREM_I16, - SREM_I32, - SREM_I64, - SREM_I128, - UREM_I8, - UREM_I16, - UREM_I32, - UREM_I64, - UREM_I128, - SDIVREM_I8, - SDIVREM_I16, - SDIVREM_I32, - SDIVREM_I64, - SDIVREM_I128, - UDIVREM_I8, - UDIVREM_I16, - UDIVREM_I32, - UDIVREM_I64, - UDIVREM_I128, - NEG_I32, - NEG_I64, - - // FLOATING POINT - ADD_F32, - ADD_F64, - ADD_F80, - ADD_F128, - ADD_PPCF128, - SUB_F32, - SUB_F64, - SUB_F80, - SUB_F128, - SUB_PPCF128, - MUL_F32, - MUL_F64, - MUL_F80, - MUL_F128, - MUL_PPCF128, - DIV_F32, - DIV_F64, - DIV_F80, - DIV_F128, - DIV_PPCF128, - REM_F32, - REM_F64, - REM_F80, - REM_F128, - REM_PPCF128, - FMA_F32, - FMA_F64, - FMA_F80, - FMA_F128, - FMA_PPCF128, - POWI_F32, - POWI_F64, - POWI_F80, - POWI_F128, - POWI_PPCF128, - SQRT_F32, - SQRT_F64, - SQRT_F80, - SQRT_F128, - SQRT_PPCF128, - LOG_F32, - LOG_F64, - LOG_F80, - LOG_F128, - LOG_PPCF128, - LOG2_F32, - LOG2_F64, - LOG2_F80, - LOG2_F128, - LOG2_PPCF128, - LOG10_F32, - LOG10_F64, - LOG10_F80, - LOG10_F128, - LOG10_PPCF128, - EXP_F32, - EXP_F64, - EXP_F80, - EXP_F128, - EXP_PPCF128, - EXP2_F32, - EXP2_F64, - EXP2_F80, - EXP2_F128, - EXP2_PPCF128, - SIN_F32, - SIN_F64, - SIN_F80, - SIN_F128, - SIN_PPCF128, - COS_F32, - COS_F64, - COS_F80, - COS_F128, - COS_PPCF128, - SINCOS_F32, - SINCOS_F64, - SINCOS_F80, - SINCOS_F128, - SINCOS_PPCF128, - POW_F32, - POW_F64, - POW_F80, - POW_F128, - POW_PPCF128, - CEIL_F32, - CEIL_F64, - CEIL_F80, - CEIL_F128, - CEIL_PPCF128, - TRUNC_F32, - TRUNC_F64, - TRUNC_F80, - TRUNC_F128, - TRUNC_PPCF128, - RINT_F32, - RINT_F64, - RINT_F80, - RINT_F128, - RINT_PPCF128, - NEARBYINT_F32, - NEARBYINT_F64, - NEARBYINT_F80, - NEARBYINT_F128, - NEARBYINT_PPCF128, - ROUND_F32, - ROUND_F64, - ROUND_F80, - ROUND_F128, - ROUND_PPCF128, - FLOOR_F32, - FLOOR_F64, - FLOOR_F80, - FLOOR_F128, - FLOOR_PPCF128, - COPYSIGN_F32, - COPYSIGN_F64, - COPYSIGN_F80, - COPYSIGN_F128, - COPYSIGN_PPCF128, - FMIN_F32, - FMIN_F64, - FMIN_F80, - FMIN_F128, - FMIN_PPCF128, - FMAX_F32, - FMAX_F64, - FMAX_F80, - FMAX_F128, - FMAX_PPCF128, - - // CONVERSION - FPEXT_F32_PPCF128, - FPEXT_F64_PPCF128, - FPEXT_F64_F128, - FPEXT_F32_F128, - FPEXT_F32_F64, - FPEXT_F16_F32, - FPROUND_F32_F16, - FPROUND_F64_F16, - FPROUND_F80_F16, - FPROUND_F128_F16, - FPROUND_PPCF128_F16, - FPROUND_F64_F32, - FPROUND_F80_F32, - FPROUND_F128_F32, - FPROUND_PPCF128_F32, - FPROUND_F80_F64, - FPROUND_F128_F64, - FPROUND_PPCF128_F64, - FPTOSINT_F32_I32, - FPTOSINT_F32_I64, - FPTOSINT_F32_I128, - FPTOSINT_F64_I32, - FPTOSINT_F64_I64, - FPTOSINT_F64_I128, - FPTOSINT_F80_I32, - FPTOSINT_F80_I64, - FPTOSINT_F80_I128, - FPTOSINT_F128_I32, - FPTOSINT_F128_I64, - FPTOSINT_F128_I128, - FPTOSINT_PPCF128_I32, - FPTOSINT_PPCF128_I64, - FPTOSINT_PPCF128_I128, - FPTOUINT_F32_I32, - FPTOUINT_F32_I64, - FPTOUINT_F32_I128, - FPTOUINT_F64_I32, - FPTOUINT_F64_I64, - FPTOUINT_F64_I128, - FPTOUINT_F80_I32, - FPTOUINT_F80_I64, - FPTOUINT_F80_I128, - FPTOUINT_F128_I32, - FPTOUINT_F128_I64, - FPTOUINT_F128_I128, - FPTOUINT_PPCF128_I32, - FPTOUINT_PPCF128_I64, - FPTOUINT_PPCF128_I128, - SINTTOFP_I32_F32, - SINTTOFP_I32_F64, - SINTTOFP_I32_F80, - SINTTOFP_I32_F128, - SINTTOFP_I32_PPCF128, - SINTTOFP_I64_F32, - SINTTOFP_I64_F64, - SINTTOFP_I64_F80, - SINTTOFP_I64_F128, - SINTTOFP_I64_PPCF128, - SINTTOFP_I128_F32, - SINTTOFP_I128_F64, - SINTTOFP_I128_F80, - SINTTOFP_I128_F128, - SINTTOFP_I128_PPCF128, - UINTTOFP_I32_F32, - UINTTOFP_I32_F64, - UINTTOFP_I32_F80, - UINTTOFP_I32_F128, - UINTTOFP_I32_PPCF128, - UINTTOFP_I64_F32, - UINTTOFP_I64_F64, - UINTTOFP_I64_F80, - UINTTOFP_I64_F128, - UINTTOFP_I64_PPCF128, - UINTTOFP_I128_F32, - UINTTOFP_I128_F64, - UINTTOFP_I128_F80, - UINTTOFP_I128_F128, - UINTTOFP_I128_PPCF128, - - // COMPARISON - OEQ_F32, - OEQ_F64, - OEQ_F128, - OEQ_PPCF128, - UNE_F32, - UNE_F64, - UNE_F128, - UNE_PPCF128, - OGE_F32, - OGE_F64, - OGE_F128, - OGE_PPCF128, - OLT_F32, - OLT_F64, - OLT_F128, - OLT_PPCF128, - OLE_F32, - OLE_F64, - OLE_F128, - OLE_PPCF128, - OGT_F32, - OGT_F64, - OGT_F128, - OGT_PPCF128, - UO_F32, - UO_F64, - UO_F128, - UO_PPCF128, - O_F32, - O_F64, - O_F128, - O_PPCF128, - - // MEMORY - MEMCPY, - MEMSET, - MEMMOVE, - - // ELEMENT-WISE UNORDERED-ATOMIC MEMORY of different element sizes - MEMCPY_ELEMENT_UNORDERED_ATOMIC_1, - MEMCPY_ELEMENT_UNORDERED_ATOMIC_2, - MEMCPY_ELEMENT_UNORDERED_ATOMIC_4, - MEMCPY_ELEMENT_UNORDERED_ATOMIC_8, - MEMCPY_ELEMENT_UNORDERED_ATOMIC_16, - - MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1, - MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2, - MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4, - MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8, - MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16, - - MEMSET_ELEMENT_UNORDERED_ATOMIC_1, - MEMSET_ELEMENT_UNORDERED_ATOMIC_2, - MEMSET_ELEMENT_UNORDERED_ATOMIC_4, - MEMSET_ELEMENT_UNORDERED_ATOMIC_8, - MEMSET_ELEMENT_UNORDERED_ATOMIC_16, - - // EXCEPTION HANDLING - UNWIND_RESUME, - - // Note: there's two sets of atomics libcalls; see - // <http://llvm.org/docs/Atomics.html> for more info on the - // difference between them. - - // Atomic '__sync_*' libcalls. - SYNC_VAL_COMPARE_AND_SWAP_1, - SYNC_VAL_COMPARE_AND_SWAP_2, - SYNC_VAL_COMPARE_AND_SWAP_4, - SYNC_VAL_COMPARE_AND_SWAP_8, - SYNC_VAL_COMPARE_AND_SWAP_16, - SYNC_LOCK_TEST_AND_SET_1, - SYNC_LOCK_TEST_AND_SET_2, - SYNC_LOCK_TEST_AND_SET_4, - SYNC_LOCK_TEST_AND_SET_8, - SYNC_LOCK_TEST_AND_SET_16, - SYNC_FETCH_AND_ADD_1, - SYNC_FETCH_AND_ADD_2, - SYNC_FETCH_AND_ADD_4, - SYNC_FETCH_AND_ADD_8, - SYNC_FETCH_AND_ADD_16, - SYNC_FETCH_AND_SUB_1, - SYNC_FETCH_AND_SUB_2, - SYNC_FETCH_AND_SUB_4, - SYNC_FETCH_AND_SUB_8, - SYNC_FETCH_AND_SUB_16, - SYNC_FETCH_AND_AND_1, - SYNC_FETCH_AND_AND_2, - SYNC_FETCH_AND_AND_4, - SYNC_FETCH_AND_AND_8, - SYNC_FETCH_AND_AND_16, - SYNC_FETCH_AND_OR_1, - SYNC_FETCH_AND_OR_2, - SYNC_FETCH_AND_OR_4, - SYNC_FETCH_AND_OR_8, - SYNC_FETCH_AND_OR_16, - SYNC_FETCH_AND_XOR_1, - SYNC_FETCH_AND_XOR_2, - SYNC_FETCH_AND_XOR_4, - SYNC_FETCH_AND_XOR_8, - SYNC_FETCH_AND_XOR_16, - SYNC_FETCH_AND_NAND_1, - SYNC_FETCH_AND_NAND_2, - SYNC_FETCH_AND_NAND_4, - SYNC_FETCH_AND_NAND_8, - SYNC_FETCH_AND_NAND_16, - SYNC_FETCH_AND_MAX_1, - SYNC_FETCH_AND_MAX_2, - SYNC_FETCH_AND_MAX_4, - SYNC_FETCH_AND_MAX_8, - SYNC_FETCH_AND_MAX_16, - SYNC_FETCH_AND_UMAX_1, - SYNC_FETCH_AND_UMAX_2, - SYNC_FETCH_AND_UMAX_4, - SYNC_FETCH_AND_UMAX_8, - SYNC_FETCH_AND_UMAX_16, - SYNC_FETCH_AND_MIN_1, - SYNC_FETCH_AND_MIN_2, - SYNC_FETCH_AND_MIN_4, - SYNC_FETCH_AND_MIN_8, - SYNC_FETCH_AND_MIN_16, - SYNC_FETCH_AND_UMIN_1, - SYNC_FETCH_AND_UMIN_2, - SYNC_FETCH_AND_UMIN_4, - SYNC_FETCH_AND_UMIN_8, - SYNC_FETCH_AND_UMIN_16, - - // Atomic '__atomic_*' libcalls. - ATOMIC_LOAD, - ATOMIC_LOAD_1, - ATOMIC_LOAD_2, - ATOMIC_LOAD_4, - ATOMIC_LOAD_8, - ATOMIC_LOAD_16, - - ATOMIC_STORE, - ATOMIC_STORE_1, - ATOMIC_STORE_2, - ATOMIC_STORE_4, - ATOMIC_STORE_8, - ATOMIC_STORE_16, - - ATOMIC_EXCHANGE, - ATOMIC_EXCHANGE_1, - ATOMIC_EXCHANGE_2, - ATOMIC_EXCHANGE_4, - ATOMIC_EXCHANGE_8, - ATOMIC_EXCHANGE_16, - - ATOMIC_COMPARE_EXCHANGE, - ATOMIC_COMPARE_EXCHANGE_1, - ATOMIC_COMPARE_EXCHANGE_2, - ATOMIC_COMPARE_EXCHANGE_4, - ATOMIC_COMPARE_EXCHANGE_8, - ATOMIC_COMPARE_EXCHANGE_16, - - ATOMIC_FETCH_ADD_1, - ATOMIC_FETCH_ADD_2, - ATOMIC_FETCH_ADD_4, - ATOMIC_FETCH_ADD_8, - ATOMIC_FETCH_ADD_16, - - ATOMIC_FETCH_SUB_1, - ATOMIC_FETCH_SUB_2, - ATOMIC_FETCH_SUB_4, - ATOMIC_FETCH_SUB_8, - ATOMIC_FETCH_SUB_16, - - ATOMIC_FETCH_AND_1, - ATOMIC_FETCH_AND_2, - ATOMIC_FETCH_AND_4, - ATOMIC_FETCH_AND_8, - ATOMIC_FETCH_AND_16, - - ATOMIC_FETCH_OR_1, - ATOMIC_FETCH_OR_2, - ATOMIC_FETCH_OR_4, - ATOMIC_FETCH_OR_8, - ATOMIC_FETCH_OR_16, - - ATOMIC_FETCH_XOR_1, - ATOMIC_FETCH_XOR_2, - ATOMIC_FETCH_XOR_4, - ATOMIC_FETCH_XOR_8, - ATOMIC_FETCH_XOR_16, - - ATOMIC_FETCH_NAND_1, - ATOMIC_FETCH_NAND_2, - ATOMIC_FETCH_NAND_4, - ATOMIC_FETCH_NAND_8, - ATOMIC_FETCH_NAND_16, - - // Stack Protector Fail. - STACKPROTECTOR_CHECK_FAIL, - - // Deoptimization. - DEOPTIMIZE, - - UNKNOWN_LIBCALL +#define HANDLE_LIBCALL(code, name) code, + #include "RuntimeLibcalls.def" +#undef HANDLE_LIBCALL }; /// getFPEXT - Return the FPEXT_*_* value for the given types, or diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h index 25afc5b506df..f3f2f05b877d 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAG.h @@ -22,8 +22,8 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/TargetLowering.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Target/TargetLowering.h" #include <cassert> #include <cstddef> #include <iterator> diff --git a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 218e22e40234..14882205584e 100644 --- a/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/contrib/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -24,9 +24,9 @@ #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/MC/LaneBitmask.h" -#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <cstdint> #include <list> @@ -275,6 +275,11 @@ namespace llvm { /// Returns an existing SUnit for this MI, or nullptr. SUnit *getSUnit(MachineInstr *MI) const; + /// If this method returns true, handling of the scheduling regions + /// themselves (in case of a scheduling boundary in MBB) will be done + /// beginning with the topmost region of MBB. + virtual bool doMBBSchedRegionsTopDown() const { return false; } + /// Prepares to perform scheduling in the given block. virtual void startBlock(MachineBasicBlock *BB); diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h index d6851f7143a5..6a5c2db34bb1 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -211,6 +211,7 @@ class SelectionDAG { const SelectionDAGTargetInfo *TSI = nullptr; const TargetLowering *TLI = nullptr; MachineFunction *MF; + Pass *SDAGISelPass = nullptr; LLVMContext *Context; CodeGenOpt::Level OptLevel; @@ -335,6 +336,14 @@ private: .getRawSubclassData(); } + template <typename SDNodeTy> + static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order, + SDVTList VTs, EVT MemoryVT, + MachineMemOperand *MMO) { + return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO) + .getRawSubclassData(); + } + void createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { assert(!Node->OperandList && "Node already has operands"); SDUse *Ops = OperandRecycler.allocate( @@ -366,13 +375,16 @@ public: ~SelectionDAG(); /// Prepare this SelectionDAG to process code in the given MachineFunction. - void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE); + void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, + Pass *PassPtr); /// Clear state and free memory necessary to make this /// SelectionDAG ready to process a new block. void clear(); MachineFunction &getMachineFunction() const { return *MF; } + const Pass *getPass() const { return SDAGISelPass; } + const DataLayout &getDataLayout() const { return MF->getDataLayout(); } const TargetMachine &getTarget() const { return TM; } const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); } @@ -631,6 +643,8 @@ public: SDValue getRegister(unsigned Reg, EVT VT); SDValue getRegisterMask(const uint32_t *RegMask); SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label); + SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, + MCSymbol *Label); SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0, bool isTarget = false, unsigned char TargetFlags = 0); @@ -782,6 +796,24 @@ public: /// \brief Create a logical NOT operation as (XOR Val, BooleanOne). SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); + /// \brief Create an add instruction with appropriate flags when used for + /// addressing some offset of an object. i.e. if a load is split into multiple + /// components, create an add nuw from the base pointer to the offset. + SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset) { + EVT VT = Op.getValueType(); + return getObjectPtrOffset(SL, Op, getConstant(Offset, SL, VT)); + } + + SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) { + EVT VT = Op.getValueType(); + + // The object itself can't wrap around the address space, so it shouldn't be + // possible for the adds of the offsets to the split parts to overflow. + SDNodeFlags Flags; + Flags.setNoUnsignedWrap(true); + return getNode(ISD::ADD, SL, VT, Op, Offset, Flags); + } + /// Return a new CALLSEQ_START node, that starts new call frame, in which /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and /// OutSize specifies part of the frame set up prior to the sequence. @@ -956,11 +988,14 @@ public: /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not /// less than FIRST_TARGET_MEMORY_OPCODE. - SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, - ArrayRef<SDValue> Ops, EVT MemVT, - MachinePointerInfo PtrInfo, unsigned Align = 0, - bool Vol = false, bool ReadMem = true, - bool WriteMem = true, unsigned Size = 0); + SDValue getMemIntrinsicNode( + unsigned Opcode, const SDLoc &dl, SDVTList VTList, + ArrayRef<SDValue> Ops, EVT MemVT, + MachinePointerInfo PtrInfo, + unsigned Align = 0, + MachineMemOperand::Flags Flags + = MachineMemOperand::MOLoad | MachineMemOperand::MOStore, + unsigned Size = 0); SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, EVT MemVT, @@ -1166,19 +1201,26 @@ public: const SDNodeFlags Flags = SDNodeFlags()); /// Creates a SDDbgValue node. - SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, - bool IsIndirect, uint64_t Off, const DebugLoc &DL, + SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, + unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O); - /// Constant - SDDbgValue *getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, - uint64_t Off, const DebugLoc &DL, unsigned O); + /// Creates a constant SDDbgValue node. + SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr, + const Value *C, const DebugLoc &DL, + unsigned O); - /// FrameIndex - SDDbgValue *getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, - uint64_t Off, const DebugLoc &DL, + /// Creates a FrameIndex SDDbgValue node. + SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, + unsigned FI, const DebugLoc &DL, unsigned O); + /// Transfer debug values from one node to another, while optionally + /// generating fragment expressions for split-up values. If \p InvalidateDbg + /// is set, debug values are invalidated after they are transferred. + void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0, + unsigned SizeInBits = 0, bool InvalidateDbg = true); + /// Remove the specified node from the system. If any of its /// operands then becomes dead, remove them as well. Inform UpdateListener /// for each node deleted. @@ -1208,7 +1250,7 @@ public: void ReplaceAllUsesWith(SDNode *From, const SDValue *To); /// Replace any uses of From with To, leaving - /// uses of other values produced by From.Val alone. + /// uses of other values produced by From.getNode() alone. void ReplaceAllUsesOfValueWith(SDValue From, SDValue To); /// Like ReplaceAllUsesOfValueWith, but for multiple values at once. @@ -1259,10 +1301,6 @@ public: return DbgInfo->getSDDbgValues(SD); } -private: - /// Transfer SDDbgValues. Called via ReplaceAllUses{OfValue}?With - void TransferDbgValues(SDValue From, SDValue To); - public: /// Return true if there are any SDDbgValue nodes associated /// with this SelectionDAG. @@ -1279,6 +1317,10 @@ public: return DbgInfo->ByvalParmDbgEnd(); } + /// To be invoked on an SDNode that is slated to be erased. This + /// function mirrors \c llvm::salvageDebugInfo. + void salvageDebugInfo(SDNode &N); + void dump() const; /// Create a stack temporary, suitable for holding the specified value type. @@ -1308,6 +1350,14 @@ public: SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl); + /// See if the specified operand can be simplified with the knowledge that only + /// the bits specified by Mask are used. If so, return the simpler operand, + /// otherwise return a null SDValue. + /// + /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can + /// simplify nodes with multiple uses more aggressively.) + SDValue GetDemandedBits(SDValue V, const APInt &Mask); + /// Return true if the sign bit of Op is known to be zero. /// We use this predicate to simplify operations downstream. bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const; diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 2107e5a31381..18e4c7a83def 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -1,5 +1,4 @@ -//===-- llvm/CodeGen/SelectionDAGAddressAnalysis.h ------- DAG Address Analysis -//---*- C++ -*-===// +//===- SelectionDAGAddressAnalysis.h - DAG Address Analysis -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,16 +6,17 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// #ifndef LLVM_CODEGEN_SELECTIONDAGADDRESSANALYSIS_H #define LLVM_CODEGEN_SELECTIONDAGADDRESSANALYSIS_H -#include "llvm/CodeGen/ISDOpcodes.h" -#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include <cstdint> namespace llvm { + +class SelectionDAG; + /// Helper struct to parse and store a memory address as base + index + offset. /// We ignore sign extensions when it is safe to do so. /// The following two expressions are not equivalent. To differentiate we need @@ -34,12 +34,11 @@ class BaseIndexOffset { private: SDValue Base; SDValue Index; - int64_t Offset; - bool IsIndexSignExt; + int64_t Offset = 0; + bool IsIndexSignExt = false; public: - BaseIndexOffset() : Offset(0), IsIndexSignExt(false) {} - + BaseIndexOffset() = default; BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset, bool IsIndexSignExt) : Base(Base), Index(Index), Offset(Offset), @@ -59,6 +58,7 @@ public: /// Parses tree in Ptr for base, index, offset addresses. static BaseIndexOffset match(SDValue Ptr, const SelectionDAG &DAG); }; -} // namespace llvm -#endif +} // end namespace llvm + +#endif // LLVM_CODEGEN_SELECTIONDAGADDRESSANALYSIS_H diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 591b2f773344..de6849a1eae1 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -17,9 +17,9 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/Pass.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include <memory> namespace llvm { @@ -130,6 +130,7 @@ public: OPC_CheckOpcode, OPC_SwitchOpcode, OPC_CheckType, + OPC_CheckTypeRes, OPC_SwitchType, OPC_CheckChild0Type, OPC_CheckChild1Type, OPC_CheckChild2Type, OPC_CheckChild3Type, OPC_CheckChild4Type, OPC_CheckChild5Type, @@ -275,6 +276,8 @@ public: return false; } + bool isOrEquivalentToAdd(const SDNode *N) const; + private: // Calls to these functions are generated by tblgen. diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 5fb69ae232af..7de2e766d521 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -85,10 +85,7 @@ namespace ISD { /// If N is a BUILD_VECTOR node whose elements are all the same constant or /// undefined, return true and return the constant value in \p SplatValue. - /// This sets \p SplatValue to the smallest possible splat unless AllowShrink - /// is set to false. - bool isConstantSplatVector(const SDNode *N, APInt &SplatValue, - bool AllowShrink = true); + bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); /// Return true if the specified node is a BUILD_VECTOR where all of the /// elements are ~0 or undef. @@ -626,13 +623,14 @@ public: /// Test if this node is a strict floating point pseudo-op. bool isStrictFPOpcode() { switch (NodeType) { - default: + default: return false; case ISD::STRICT_FADD: case ISD::STRICT_FSUB: case ISD::STRICT_FMUL: case ISD::STRICT_FDIV: case ISD::STRICT_FREM: + case ISD::STRICT_FMA: case ISD::STRICT_FSQRT: case ISD::STRICT_FPOW: case ISD::STRICT_FPOWI: @@ -1436,6 +1434,9 @@ public: const APInt &getAPIntValue() const { return Value->getValue(); } uint64_t getZExtValue() const { return Value->getZExtValue(); } int64_t getSExtValue() const { return Value->getSExtValue(); } + uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) { + return Value->getLimitedValue(Limit); + } bool isOne() const { return Value->isOne(); } bool isNullValue() const { return Value->isZero(); } @@ -1489,11 +1490,7 @@ public: /// convenient to write "2.0" and the like. Without this function we'd /// have to duplicate its logic everywhere it's called. bool isExactlyValue(double V) const { - bool ignored; - APFloat Tmp(V); - Tmp.convert(Value->getValueAPF().getSemantics(), - APFloat::rmNearestTiesToEven, &ignored); - return isExactlyValue(Tmp); + return Value->getValueAPF().isExactlyValue(V); } bool isExactlyValue(const APFloat& V) const; @@ -1850,19 +1847,20 @@ public: } }; -class EHLabelSDNode : public SDNode { +class LabelSDNode : public SDNode { friend class SelectionDAG; MCSymbol *Label; - EHLabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L) + LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L) : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} public: MCSymbol *getLabel() const { return Label; } static bool classof(const SDNode *N) { - return N->getOpcode() == ISD::EH_LABEL; + return N->getOpcode() == ISD::EH_LABEL || + N->getOpcode() == ISD::ANNOTATION_LABEL; } }; @@ -2017,6 +2015,9 @@ public: /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } + void setTruncatingStore(bool Truncating) { + StoreSDNodeBits.IsTruncating = Truncating; + } const SDValue &getValue() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(2); } @@ -2113,7 +2114,7 @@ class MaskedGatherScatterSDNode : public MemSDNode { public: friend class SelectionDAG; - MaskedGatherScatterSDNode(unsigned NodeTy, unsigned Order, + MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} diff --git a/contrib/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/contrib/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h index ac5092af8def..45c1df48a5e6 100644 --- a/contrib/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h @@ -1,4 +1,4 @@ -//==-- llvm/CodeGen/SelectionDAGTargetInfo.h - SelectionDAG Info -*- C++ -*-==// +//==- llvm/CodeGen/SelectionDAGTargetInfo.h - SelectionDAG Info --*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -16,21 +16,24 @@ #ifndef LLVM_CODEGEN_SELECTIONDAGTARGETINFO_H #define LLVM_CODEGEN_SELECTIONDAGTARGETINFO_H +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/Support/CodeGen.h" +#include <utility> namespace llvm { +class SelectionDAG; + //===----------------------------------------------------------------------===// /// Targets can subclass this to parameterize the /// SelectionDAG lowering and instruction selection process. /// class SelectionDAGTargetInfo { - SelectionDAGTargetInfo(const SelectionDAGTargetInfo &) = delete; - void operator=(const SelectionDAGTargetInfo &) = delete; - public: explicit SelectionDAGTargetInfo() = default; + SelectionDAGTargetInfo(const SelectionDAGTargetInfo &) = delete; + SelectionDAGTargetInfo &operator=(const SelectionDAGTargetInfo &) = delete; virtual ~SelectionDAGTargetInfo(); /// Emit target-specific code that performs a memcpy. @@ -144,6 +147,7 @@ public: MachinePointerInfo SrcPtrInfo) const { return std::make_pair(SDValue(), SDValue()); } + // Return true when the decision to generate FMA's (or FMS, FMLA etc) rather // than FMUL and ADD is delegated to the machine combiner. virtual bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const { @@ -151,6 +155,6 @@ public: } }; -} // end llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_SELECTIONDAGTARGETINFO_H diff --git a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h index a7b16e7a9ed2..3a91e363f923 100644 --- a/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h +++ b/contrib/llvm/include/llvm/CodeGen/SlotIndexes.h @@ -139,7 +139,7 @@ class raw_ostream; }; /// Construct an invalid index. - SlotIndex() : lie(nullptr, 0) {} + SlotIndex() = default; // Construct a new slot index from the given one, and set the slot. SlotIndex(const SlotIndex &li, Slot s) : lie(li.listEntry(), unsigned(s)) { diff --git a/contrib/llvm/include/llvm/CodeGen/StackMaps.h b/contrib/llvm/include/llvm/CodeGen/StackMaps.h index 8263946ed928..4407114d2741 100644 --- a/contrib/llvm/include/llvm/CodeGen/StackMaps.h +++ b/contrib/llvm/include/llvm/CodeGen/StackMaps.h @@ -14,6 +14,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/IR/CallingConv.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/Debug.h" #include <algorithm> #include <cassert> @@ -25,7 +26,6 @@ namespace llvm { class AsmPrinter; class MCExpr; class MCStreamer; -class MCSymbol; class raw_ostream; class TargetRegisterInfo; diff --git a/contrib/llvm/include/llvm/CodeGen/TailDuplicator.h b/contrib/llvm/include/llvm/CodeGen/TailDuplicator.h index 483c0ab1eec9..be6562c85f2e 100644 --- a/contrib/llvm/include/llvm/CodeGen/TailDuplicator.h +++ b/contrib/llvm/include/llvm/CodeGen/TailDuplicator.h @@ -19,11 +19,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include <utility> #include <vector> @@ -61,13 +57,14 @@ class TailDuplicator { public: /// Prepare to run on a specific machine function. /// @param MF - Function that will be processed + /// @param PreRegAlloc - true if used before register allocation /// @param MBPI - Branch Probability Info. Used to propagate correct /// probabilities when modifying the CFG. /// @param LayoutMode - When true, don't use the existing layout to make /// decisions. /// @param TailDupSize - Maxmimum size of blocks to tail-duplicate. Zero /// default implies using the command line value TailDupSize. - void initMF(MachineFunction &MF, + void initMF(MachineFunction &MF, bool PreRegAlloc, const MachineBranchProbabilityInfo *MBPI, bool LayoutMode, unsigned TailDupSize = 0); diff --git a/contrib/llvm/include/llvm/Target/TargetCallingConv.h b/contrib/llvm/include/llvm/CodeGen/TargetCallingConv.h index 4f750b8a289f..8646a15599cb 100644 --- a/contrib/llvm/include/llvm/Target/TargetCallingConv.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetCallingConv.h - Calling Convention ----*- C++ -*-===// +//===-- llvm/CodeGen/TargetCallingConv.h - Calling Convention ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETCALLINGCONV_H -#define LLVM_TARGET_TARGETCALLINGCONV_H +#ifndef LLVM_CODEGEN_TARGETCALLINGCONV_H +#define LLVM_CODEGEN_TARGETCALLINGCONV_H #include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/ValueTypes.h" @@ -201,4 +201,4 @@ namespace ISD { } // end namespace ISD } // end namespace llvm -#endif // LLVM_TARGET_TARGETCALLINGCONV_H +#endif // LLVM_CODEGEN_TARGETCALLINGCONV_H diff --git a/contrib/llvm/include/llvm/Target/TargetFrameLowering.h b/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h index 4576f8c7582b..61f1cf07bcf2 100644 --- a/contrib/llvm/include/llvm/Target/TargetFrameLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetFrameLowering.h @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetFrameLowering.h ---------------------------*- C++ -*-===// +//===-- llvm/CodeGen/TargetFrameLowering.h ----------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETFRAMELOWERING_H -#define LLVM_TARGET_TARGETFRAMELOWERING_H +#ifndef LLVM_CODEGEN_TARGETFRAMELOWERING_H +#define LLVM_CODEGEN_TARGETFRAMELOWERING_H #include "llvm/CodeGen/MachineBasicBlock.h" #include <utility> @@ -193,10 +193,12 @@ public: /// restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee /// saved registers and returns true if it isn't possible / profitable to do /// so by issuing a series of load instructions via loadRegToStackSlot(). + /// If it returns true, and any of the registers in CSI is not restored, + /// it sets the corresponding Restored flag in CSI to false. /// Returns false otherwise. virtual bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const std::vector<CalleeSavedInfo> &CSI, + std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const { return false; } @@ -328,12 +330,12 @@ public: /// Check if given function is safe for not having callee saved registers. /// This is used when interprocedural register allocation is enabled. - static bool isSafeForNoCSROpt(const Function *F) { - if (!F->hasLocalLinkage() || F->hasAddressTaken() || - !F->hasFnAttribute(Attribute::NoRecurse)) + static bool isSafeForNoCSROpt(const Function &F) { + if (!F.hasLocalLinkage() || F.hasAddressTaken() || + !F.hasFnAttribute(Attribute::NoRecurse)) return false; // Function should not be optimized as tail call. - for (const User *U : F->users()) + for (const User *U : F.users()) if (auto CS = ImmutableCallSite(U)) if (CS.isTailCall()) return false; diff --git a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 1843a2eed9bf..38a1b33aecad 100644 --- a/contrib/llvm/include/llvm/Target/TargetInstrInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1,4 +1,4 @@ -//===- llvm/Target/TargetInstrInfo.h - Instruction Info ---------*- C++ -*-===// +//===- llvm/CodeGen/TargetInstrInfo.h - Instruction Info --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -18,13 +18,13 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/None.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/ErrorHandling.h" @@ -38,6 +38,7 @@ namespace llvm { class DFAPacketizer; class InstrItineraryData; +class LiveIntervals; class LiveVariables; class MachineMemOperand; class MachineRegisterInfo; @@ -55,7 +56,7 @@ class TargetRegisterInfo; class TargetSchedModel; class TargetSubtargetInfo; -template<class T> class SmallVectorImpl; +template <class T> class SmallVectorImpl; //--------------------------------------------------------------------------- /// @@ -66,8 +67,7 @@ public: TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u, unsigned CatchRetOpcode = ~0u, unsigned ReturnOpcode = ~0u) : CallFrameSetupOpcode(CFSetupOpcode), - CallFrameDestroyOpcode(CFDestroyOpcode), - CatchRetOpcode(CatchRetOpcode), + CallFrameDestroyOpcode(CFDestroyOpcode), CatchRetOpcode(CatchRetOpcode), ReturnOpcode(ReturnOpcode) {} TargetInstrInfo(const TargetInstrInfo &) = delete; TargetInstrInfo &operator=(const TargetInstrInfo &) = delete; @@ -79,8 +79,7 @@ public: /// Given a machine instruction descriptor, returns the register /// class constraint for OpNum, or NULL. - const TargetRegisterClass *getRegClass(const MCInstrDesc &TID, - unsigned OpNum, + const TargetRegisterClass *getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const; @@ -139,8 +138,7 @@ protected: /// the fixed result pair is equal to or equivalent to the source pair of /// indices: (CommutableOpIdx1, CommutableOpIdx2). It is assumed here that /// the pairs (x,y) and (y,x) are equivalent. - static bool fixCommutedOpIndices(unsigned &ResultIdx1, - unsigned &ResultIdx2, + static bool fixCommutedOpIndices(unsigned &ResultIdx1, unsigned &ResultIdx2, unsigned CommutableOpIdx1, unsigned CommutableOpIdx2); @@ -164,7 +162,7 @@ public: /// Returns true if the argument is a frame pseudo instruction. bool isFrameInstr(const MachineInstr &I) const { return I.getOpcode() == getCallFrameSetupOpcode() || - I.getOpcode() == getCallFrameDestroyOpcode(); + I.getOpcode() == getCallFrameDestroyOpcode(); } /// Returns true if the argument is a frame setup pseudo instruction. @@ -191,7 +189,8 @@ public: /// prior to the pair. int64_t getFrameTotalSize(const MachineInstr &I) const { if (isFrameSetup(I)) { - assert(I.getOperand(1).getImm() >= 0 && "Frame size must not be negative"); + assert(I.getOperand(1).getImm() >= 0 && + "Frame size must not be negative"); return getFrameSize(I) + I.getOperand(1).getImm(); } return getFrameSize(I); @@ -211,9 +210,8 @@ public: /// destination. e.g. X86::MOVSX64rr32. If this returns true, then it's /// expected the pre-extension value is available as a subreg of the result /// register. This also returns the sub-register index in SubIdx. - virtual bool isCoalescableExtInstr(const MachineInstr &MI, - unsigned &SrcReg, unsigned &DstReg, - unsigned &SubIdx) const { + virtual bool isCoalescableExtInstr(const MachineInstr &MI, unsigned &SrcReg, + unsigned &DstReg, unsigned &SubIdx) const { return false; } @@ -315,9 +313,7 @@ public: /// MachineSink determines on its own whether the instruction is safe to sink; /// this gives the target a hook to override the default behavior with regards /// to which instructions should be sunk. - virtual bool shouldSink(const MachineInstr &MI) const { - return true; - } + virtual bool shouldSink(const MachineInstr &MI) const { return true; } /// Re-issue the specified 'original' instruction at the /// specific location targeting a new destination register. @@ -329,13 +325,14 @@ public: unsigned SubIdx, const MachineInstr &Orig, const TargetRegisterInfo &TRI) const; - /// Create a duplicate of the Orig instruction in MF. This is like - /// MachineFunction::CloneMachineInstr(), but the target may update operands + /// \brief Clones instruction or the whole instruction bundle \p Orig and + /// insert into \p MBB before \p InsertBefore. The target may update operands /// that are required to be unique. /// - /// The instruction must be duplicable as indicated by isNotDuplicable(). - virtual MachineInstr *duplicate(MachineInstr &Orig, - MachineFunction &MF) const; + /// \p Orig must not return true for MachineInstr::isNotDuplicable(). + virtual MachineInstr &duplicate(MachineBasicBlock &MBB, + MachineBasicBlock::iterator InsertBefore, + const MachineInstr &Orig) const; /// This method must be implemented by targets that /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target @@ -425,10 +422,10 @@ public: /// and \p DefIdx. /// \p [out] InputRegs of the equivalent REG_SEQUENCE. Each element of /// the list is modeled as <Reg:SubReg, SubIdx>. - /// E.g., REG_SEQUENCE vreg1:sub1, sub0, vreg2, sub1 would produce + /// E.g., REG_SEQUENCE %1:sub1, sub0, %2, sub1 would produce /// two elements: - /// - vreg1:sub1, sub0 - /// - vreg2<:0>, sub1 + /// - %1:sub1, sub0 + /// - %2<:0>, sub1 /// /// \returns true if it is possible to build such an input sequence /// with the pair \p MI, \p DefIdx. False otherwise. @@ -445,8 +442,8 @@ public: /// Build the equivalent inputs of a EXTRACT_SUBREG for the given \p MI /// and \p DefIdx. /// \p [out] InputReg of the equivalent EXTRACT_SUBREG. - /// E.g., EXTRACT_SUBREG vreg1:sub1, sub0, sub1 would produce: - /// - vreg1:sub1, sub0 + /// E.g., EXTRACT_SUBREG %1:sub1, sub0, sub1 would produce: + /// - %1:sub1, sub0 /// /// \returns true if it is possible to build such an input sequence /// with the pair \p MI, \p DefIdx. False otherwise. @@ -456,17 +453,16 @@ public: /// \note The generic implementation does not provide any support for /// MI.isExtractSubregLike(). In other words, one has to override /// getExtractSubregLikeInputs for target specific instructions. - bool - getExtractSubregInputs(const MachineInstr &MI, unsigned DefIdx, - RegSubRegPairAndIdx &InputReg) const; + bool getExtractSubregInputs(const MachineInstr &MI, unsigned DefIdx, + RegSubRegPairAndIdx &InputReg) const; /// Build the equivalent inputs of a INSERT_SUBREG for the given \p MI /// and \p DefIdx. /// \p [out] BaseReg and \p [out] InsertedReg contain /// the equivalent inputs of INSERT_SUBREG. - /// E.g., INSERT_SUBREG vreg0:sub0, vreg1:sub1, sub3 would produce: - /// - BaseReg: vreg0:sub0 - /// - InsertedReg: vreg1:sub1, sub3 + /// E.g., INSERT_SUBREG %0:sub0, %1:sub1, sub3 would produce: + /// - BaseReg: %0:sub0 + /// - InsertedReg: %1:sub1, sub3 /// /// \returns true if it is possible to build such an input sequence /// with the pair \p MI, \p DefIdx. False otherwise. @@ -476,10 +472,9 @@ public: /// \note The generic implementation does not provide any support for /// MI.isInsertSubregLike(). In other words, one has to override /// getInsertSubregLikeInputs for target specific instructions. - bool - getInsertSubregInputs(const MachineInstr &MI, unsigned DefIdx, - RegSubRegPair &BaseReg, - RegSubRegPairAndIdx &InsertedReg) const; + bool getInsertSubregInputs(const MachineInstr &MI, unsigned DefIdx, + RegSubRegPair &BaseReg, + RegSubRegPairAndIdx &InsertedReg) const; /// Return true if two machine instructions would produce identical values. /// By default, this is only true when the two instructions @@ -552,7 +547,7 @@ public: /// Represents a predicate at the MachineFunction level. The control flow a /// MachineBranchPredicate represents is: /// - /// Reg <def>= LHS `Predicate` RHS == ConditionDef + /// Reg = LHS `Predicate` RHS == ConditionDef /// if Reg then goto TrueDest else goto FalseDest /// struct MachineBranchPredicate { @@ -625,8 +620,8 @@ public: MachineBasicBlock *DestBB, const DebugLoc &DL, int *BytesAdded = nullptr) const { - return insertBranch(MBB, DestBB, nullptr, - ArrayRef<MachineOperand>(), DL, BytesAdded); + return insertBranch(MBB, DestBB, nullptr, ArrayRef<MachineOperand>(), DL, + BytesAdded); } /// Analyze the loop code, return true if it cannot be understoo. Upon @@ -641,8 +636,8 @@ public: /// finished. Return the value/register of the the new loop count. We need /// this function when peeling off one or more iterations of a loop. This /// function assumes the nth iteration is peeled first. - virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, - MachineInstr *IndVar, MachineInstr &Cmp, + virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineInstr *IndVar, + MachineInstr &Cmp, SmallVectorImpl<MachineOperand> &Cond, SmallVectorImpl<MachineInstr *> &PrevInsts, unsigned Iter, unsigned MaxIter) const { @@ -667,10 +662,9 @@ public: /// of the specified basic block, where the probability of the instructions /// being executed is given by Probability, and Confidence is a measure /// of our confidence that it will be properly predicted. - virtual - bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, - unsigned ExtraPredCycles, - BranchProbability Probability) const { + virtual bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, + unsigned ExtraPredCycles, + BranchProbability Probability) const { return false; } @@ -680,12 +674,11 @@ public: /// predicates, where the probability of the true path being taken is given /// by Probability, and Confidence is a measure of our confidence that it /// will be properly predicted. - virtual bool - isProfitableToIfCvt(MachineBasicBlock &TMBB, - unsigned NumTCycles, unsigned ExtraTCycles, - MachineBasicBlock &FMBB, - unsigned NumFCycles, unsigned ExtraFCycles, - BranchProbability Probability) const { + virtual bool isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumTCycles, + unsigned ExtraTCycles, + MachineBasicBlock &FMBB, unsigned NumFCycles, + unsigned ExtraFCycles, + BranchProbability Probability) const { return false; } @@ -695,9 +688,9 @@ public: /// The probability of the instructions being executed is given by /// Probability, and Confidence is a measure of our confidence that it /// will be properly predicted. - virtual bool - isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, - BranchProbability Probability) const { + virtual bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, + unsigned NumCycles, + BranchProbability Probability) const { return false; } @@ -735,9 +728,8 @@ public: /// @param TrueCycles Latency from TrueReg to select output. /// @param FalseCycles Latency from FalseReg to select output. virtual bool canInsertSelect(const MachineBasicBlock &MBB, - ArrayRef<MachineOperand> Cond, - unsigned TrueReg, unsigned FalseReg, - int &CondCycles, + ArrayRef<MachineOperand> Cond, unsigned TrueReg, + unsigned FalseReg, int &CondCycles, int &TrueCycles, int &FalseCycles) const { return false; } @@ -953,8 +945,7 @@ public: /// Set special operand attributes for new instructions after reassociation. virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, - MachineInstr &NewMI2) const { - } + MachineInstr &NewMI2) const {} /// Return true when a target supports MachineCombiner. virtual bool useMachineCombiner() const { return false; } @@ -1007,9 +998,9 @@ protected: /// \pre MI.isExtractSubregLike(). /// /// \see TargetInstrInfo::getExtractSubregInputs. - virtual bool getExtractSubregLikeInputs( - const MachineInstr &MI, unsigned DefIdx, - RegSubRegPairAndIdx &InputReg) const { + virtual bool getExtractSubregLikeInputs(const MachineInstr &MI, + unsigned DefIdx, + RegSubRegPairAndIdx &InputReg) const { return false; } @@ -1029,6 +1020,13 @@ protected: } public: + /// getAddressSpaceForPseudoSourceKind - Given the kind of memory + /// (e.g. stack) the target returns the corresponding address space. + virtual unsigned + getAddressSpaceForPseudoSourceKind(PseudoSourceValue::PSVKind Kind) const { + return 0; + } + /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. @@ -1040,7 +1038,7 @@ public: } virtual bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, - SmallVectorImpl<SDNode*> &NewNodes) const { + SmallVectorImpl<SDNode *> &NewNodes) const { return false; } @@ -1050,9 +1048,9 @@ public: /// possible. If LoadRegIndex is non-null, it is filled in with the operand /// index of the operand which will hold the register holding the loaded /// value. - virtual unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, - bool UnfoldLoad, bool UnfoldStore, - unsigned *LoadRegIndex = nullptr) const { + virtual unsigned + getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, + unsigned *LoadRegIndex = nullptr) const { return 0; } @@ -1061,7 +1059,8 @@ public: /// pointers are the same and the only differences between the two addresses /// are the offset. It also returns the offsets by reference. virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, - int64_t &Offset1, int64_t &Offset2) const { + int64_t &Offset1, + int64_t &Offset2) const { return false; } @@ -1107,16 +1106,16 @@ public: /// or /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); /// to TargetPassConfig::createMachineScheduler() to have an effect. - virtual bool shouldClusterMemOps(MachineInstr &FirstLdSt, - MachineInstr &SecondLdSt, + virtual bool shouldClusterMemOps(MachineInstr &FirstLdSt, unsigned BaseReg1, + MachineInstr &SecondLdSt, unsigned BaseReg2, unsigned NumLoads) const { llvm_unreachable("target did not implement shouldClusterMemOps()"); } /// Reverses the branch condition of the specified condition list, /// returning false on success and true if it cannot be reversed. - virtual - bool reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { + virtual bool + reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { return true; } @@ -1128,14 +1127,10 @@ public: virtual void getNoop(MCInst &NopInst) const; /// Return true for post-incremented instructions. - virtual bool isPostIncrement(const MachineInstr &MI) const { - return false; - } + virtual bool isPostIncrement(const MachineInstr &MI) const { return false; } /// Returns true if the instruction is already predicated. - virtual bool isPredicated(const MachineInstr &MI) const { - return false; - } + virtual bool isPredicated(const MachineInstr &MI) const { return false; } /// Returns true if the instruction is a /// terminator instruction that has not been predicated. @@ -1147,9 +1142,8 @@ public: } /// Returns true if the tail call can be made conditional on BranchCond. - virtual bool - canMakeTailCallConditional(SmallVectorImpl<MachineOperand> &Cond, - const MachineInstr &TailCall) const { + virtual bool canMakeTailCallConditional(SmallVectorImpl<MachineOperand> &Cond, + const MachineInstr &TailCall) const { return false; } @@ -1167,9 +1161,8 @@ public: /// Returns true if the first specified predicate /// subsumes the second, e.g. GE subsumes GT. - virtual - bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1, - ArrayRef<MachineOperand> Pred2) const { + virtual bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1, + ArrayRef<MachineOperand> Pred2) const { return false; } @@ -1207,25 +1200,25 @@ public: /// Allocate and return a hazard recognizer to use for this target when /// scheduling the machine instructions before register allocation. - virtual ScheduleHazardRecognizer* + virtual ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const; /// Allocate and return a hazard recognizer to use for this target when /// scheduling the machine instructions before register allocation. - virtual ScheduleHazardRecognizer* - CreateTargetMIHazardRecognizer(const InstrItineraryData*, + virtual ScheduleHazardRecognizer * + CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAG *DAG) const; /// Allocate and return a hazard recognizer to use for this target when /// scheduling the machine instructions after register allocation. - virtual ScheduleHazardRecognizer* - CreateTargetPostRAHazardRecognizer(const InstrItineraryData*, + virtual ScheduleHazardRecognizer * + CreateTargetPostRAHazardRecognizer(const InstrItineraryData *, const ScheduleDAG *DAG) const; /// Allocate and return a hazard recognizer to use for by non-scheduling /// passes. - virtual ScheduleHazardRecognizer* + virtual ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { return nullptr; } @@ -1439,7 +1432,7 @@ public: /// For example, AVX instructions may copy part of a register operand into /// the unused high bits of the destination register. /// - /// vcvtsi2sdq %rax, %xmm0<undef>, %xmm14 + /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 /// /// In the code above, vcvtsi2sdq copies %xmm0[127:64] into %xmm14 creating a /// false dependence on any previous write to %xmm0. @@ -1502,7 +1495,7 @@ public: /// \brief Return the value to use for the MachineCSE's LookAheadLimit, /// which is a heuristic used for CSE'ing phys reg defs. - virtual unsigned getMachineCSELookAheadLimit () const { + virtual unsigned getMachineCSELookAheadLimit() const { // The default lookahead is small to prevent unprofitable quadratic // behavior. return 5; @@ -1569,13 +1562,41 @@ public: return false; } - /// \brief Return how many instructions would be saved by outlining a - /// sequence containing \p SequenceSize instructions that appears - /// \p Occurrences times in a module. - virtual unsigned getOutliningBenefit(size_t SequenceSize, size_t Occurrences, - bool CanBeTailCall) const { + /// \brief Describes the number of instructions that it will take to call and + /// construct a frame for a given outlining candidate. + struct MachineOutlinerInfo { + /// Number of instructions to call an outlined function for this candidate. + unsigned CallOverhead; + + /// \brief Number of instructions to construct an outlined function frame + /// for this candidate. + unsigned FrameOverhead; + + /// \brief Represents the specific instructions that must be emitted to + /// construct a call to this candidate. + unsigned CallConstructionID; + + /// \brief Represents the specific instructions that must be emitted to + /// construct a frame for this candidate's outlined function. + unsigned FrameConstructionID; + + MachineOutlinerInfo() {} + MachineOutlinerInfo(unsigned CallOverhead, unsigned FrameOverhead, + unsigned CallConstructionID, + unsigned FrameConstructionID) + : CallOverhead(CallOverhead), FrameOverhead(FrameOverhead), + CallConstructionID(CallConstructionID), + FrameConstructionID(FrameConstructionID) {} + }; + + /// \brief Returns a \p MachineOutlinerInfo struct containing target-specific + /// information for a set of outlining candidates. + virtual MachineOutlinerInfo getOutlininingCandidateInfo( + std::vector< + std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>> + &RepeatedSequenceLocs) const { llvm_unreachable( - "Target didn't implement TargetInstrInfo::getOutliningBenefit!"); + "Target didn't implement TargetInstrInfo::getOutliningOverhead!"); } /// Represents how an instruction should be mapped by the outliner. @@ -1583,7 +1604,7 @@ public: /// \p Illegal instructions are those which cannot be outlined. /// \p Invisible instructions are instructions which can be outlined, but /// shouldn't actually impact the outlining result. - enum MachineOutlinerInstrType {Legal, Illegal, Invisible}; + enum MachineOutlinerInstrType { Legal, Illegal, Invisible }; /// Returns how or if \p MI should be outlined. virtual MachineOutlinerInstrType getOutliningType(MachineInstr &MI) const { @@ -1596,7 +1617,7 @@ public: /// emitted. virtual void insertOutlinerEpilogue(MachineBasicBlock &MBB, MachineFunction &MF, - bool IsTailCall) const { + const MachineOutlinerInfo &MInfo) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::insertOutlinerEpilogue!"); } @@ -1607,7 +1628,7 @@ public: virtual MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - bool IsTailCall) const { + const MachineOutlinerInfo &MInfo) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::insertOutlinedCall!"); } @@ -1616,14 +1637,17 @@ public: /// This may be empty, in which case no prologue will be emitted. virtual void insertOutlinerPrologue(MachineBasicBlock &MBB, MachineFunction &MF, - bool IsTailCall) const { + const MachineOutlinerInfo &MInfo) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::insertOutlinerPrologue!"); } /// Return true if the function can safely be outlined from. - /// By default, this means that the function has no red zone. - virtual bool isFunctionSafeToOutlineFrom(MachineFunction &MF) const { + /// A function \p MF is considered safe for outlining if an outlined function + /// produced from instructions in F will produce a program which produces the + /// same output for any set of given inputs. + virtual bool isFunctionSafeToOutlineFrom(MachineFunction &MF, + bool OutlineFromLinkOnceODRs) const { llvm_unreachable("Target didn't implement " "TargetInstrInfo::isFunctionSafeToOutlineFrom!"); } @@ -1635,25 +1659,23 @@ private: }; /// \brief Provide DenseMapInfo for TargetInstrInfo::RegSubRegPair. -template<> -struct DenseMapInfo<TargetInstrInfo::RegSubRegPair> { +template <> struct DenseMapInfo<TargetInstrInfo::RegSubRegPair> { using RegInfo = DenseMapInfo<unsigned>; static inline TargetInstrInfo::RegSubRegPair getEmptyKey() { return TargetInstrInfo::RegSubRegPair(RegInfo::getEmptyKey(), - RegInfo::getEmptyKey()); + RegInfo::getEmptyKey()); } static inline TargetInstrInfo::RegSubRegPair getTombstoneKey() { return TargetInstrInfo::RegSubRegPair(RegInfo::getTombstoneKey(), - RegInfo::getTombstoneKey()); + RegInfo::getTombstoneKey()); } /// \brief Reuse getHashValue implementation from /// std::pair<unsigned, unsigned>. static unsigned getHashValue(const TargetInstrInfo::RegSubRegPair &Val) { - std::pair<unsigned, unsigned> PairVal = - std::make_pair(Val.Reg, Val.SubReg); + std::pair<unsigned, unsigned> PairVal = std::make_pair(Val.Reg, Val.SubReg); return DenseMapInfo<std::pair<unsigned, unsigned>>::getHashValue(PairVal); } diff --git a/contrib/llvm/include/llvm/Target/TargetLowering.h b/contrib/llvm/include/llvm/CodeGen/TargetLowering.h index 23711d636c9a..0fa19d09e776 100644 --- a/contrib/llvm/include/llvm/Target/TargetLowering.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLowering.h @@ -1,4 +1,4 @@ -//===- llvm/Target/TargetLowering.h - Target Lowering Info ------*- C++ -*-===// +//===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -20,8 +20,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETLOWERING_H -#define LLVM_TARGET_TARGETLOWERING_H +#ifndef LLVM_CODEGEN_TARGETLOWERING_H +#define LLVM_CODEGEN_TARGETLOWERING_H #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" @@ -35,6 +35,7 @@ #include "llvm/CodeGen/RuntimeLibcalls.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallSite.h" @@ -51,7 +52,6 @@ #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Target/TargetCallingConv.h" #include "llvm/Target/TargetMachine.h" #include <algorithm> #include <cassert> @@ -410,9 +410,10 @@ public: return false; } - /// Should we merge stores after Legalization (generally - /// better quality) or before (simpler) - virtual bool mergeStoresAfterLegalization() const { return false; } + /// Allow store merging after legalization in addition to before legalization. + /// This may catch stores that do not exist earlier (eg, stores created from + /// intrinsics). + virtual bool mergeStoresAfterLegalization() const { return true; } /// Returns if it's reasonable to merge stores to MemVT size. virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, @@ -701,15 +702,16 @@ public: struct IntrinsicInfo { unsigned opc = 0; // target opcode EVT memVT; // memory VT - const Value* ptrVal = nullptr; // value representing memory location + + // value representing memory location + PointerUnion<const Value *, const PseudoSourceValue *> ptrVal; + int offset = 0; // offset off of ptrVal unsigned size = 0; // the size of the memory location // (taken from memVT if zero) unsigned align = 1; // alignment - bool vol = false; // is volatile? - bool readMem = false; // reads memory? - bool writeMem = false; // writes memory? + MachineMemOperand::Flags flags = MachineMemOperand::MONone; IntrinsicInfo() = default; }; @@ -718,6 +720,7 @@ public: /// true and store the intrinsic information into the IntrinsicInfo that was /// passed to the function. virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, + MachineFunction &, unsigned /*Intrinsic*/) const { return false; } @@ -733,8 +736,7 @@ public: /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to be /// legal. - virtual bool isShuffleMaskLegal(const SmallVectorImpl<int> &/*Mask*/, - EVT /*VT*/) const { + virtual bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const { return true; } @@ -791,11 +793,10 @@ public: getOperationAction(Op, VT) == Promote); } - /// Return true if the specified operation is illegal but has a custom lowering - /// on that type. This is used to help guide high-level lowering - /// decisions. + /// Return true if the operation uses custom lowering, regardless of whether + /// the type is legal or not. bool isOperationCustom(unsigned Op, EVT VT) const { - return (!isTypeLegal(VT) && getOperationAction(Op, VT) == Custom); + return getOperationAction(Op, VT) == Custom; } /// Return true if lowering to a jump table is allowed. @@ -1361,6 +1362,12 @@ public: /// getIRStackGuard returns nullptr. virtual Value *getSDagStackGuard(const Module &M) const; + /// If this function returns true, stack protection checks should XOR the + /// frame pointer (or whichever pointer is used to address locals) into the + /// stack guard value before checking it. getIRStackGuard must return nullptr + /// if this returns true. + virtual bool useStackGuardXorFP() const { return false; } + /// If the target has a standard stack protection check function that /// performs validation and error handling, returns the function. Otherwise, /// returns nullptr. Must be previously inserted by insertSSPDeclarations. @@ -1435,6 +1442,9 @@ public: /// require a more complex expansion. unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; } + /// Whether the target supports unaligned atomic operations. + bool supportsUnalignedAtomics() const { return SupportsUnalignedAtomics; } + /// Whether AtomicExpandPass should automatically insert fences and reduce /// ordering for this atomic. This should be true for most architectures with /// weak memory ordering. Defaults to false. @@ -1592,7 +1602,7 @@ public: /// Return true if a select of constants (select Cond, C1, C2) should be /// transformed into simple math ops with the condition value. For example: /// select Cond, C1, C1-1 --> add (zext Cond), C1-1 - virtual bool convertSelectOfConstantsToMath() const { + virtual bool convertSelectOfConstantsToMath(EVT VT) const { return false; } @@ -1840,11 +1850,16 @@ protected: MaxAtomicSizeInBitsSupported = SizeInBits; } - // Sets the minimum cmpxchg or ll/sc size supported by the backend. + /// Sets the minimum cmpxchg or ll/sc size supported by the backend. void setMinCmpXchgSizeInBits(unsigned SizeInBits) { MinCmpXchgSizeInBits = SizeInBits; } + /// Sets whether unaligned atomic operations are supported. + void setSupportsUnalignedAtomics(bool UnalignedSupported) { + SupportsUnalignedAtomics = UnalignedSupported; + } + public: //===--------------------------------------------------------------------===// // Addressing mode description hooks (used by LSR etc). @@ -1887,7 +1902,8 @@ public: /// /// TODO: Remove default argument virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, - Type *Ty, unsigned AddrSpace) const; + Type *Ty, unsigned AddrSpace, + Instruction *I = nullptr) const; /// \brief Return the cost of the scaling factor used in the addressing mode /// represented by AM for this target, for a load/store of the specified type. @@ -1904,10 +1920,6 @@ public: return -1; } - virtual bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) const { - return true; - } - /// Return true if the specified immediate is legal icmp immediate, that is /// the target has icmp instructions which can compare a register against the /// immediate without having to materialize the immediate into a register. @@ -1997,7 +2009,8 @@ public: bool isExtFree(const Instruction *I) const { switch (I->getOpcode()) { case Instruction::FPExt: - if (isFPExtFree(EVT::getEVT(I->getType()))) + if (isFPExtFree(EVT::getEVT(I->getType()), + EVT::getEVT(I->getOperand(0)->getType()))) return true; break; case Instruction::ZExt: @@ -2124,11 +2137,21 @@ public: /// Return true if an fpext operation is free (for instance, because /// single-precision floating-point numbers are implicitly extended to /// double-precision). - virtual bool isFPExtFree(EVT VT) const { - assert(VT.isFloatingPoint()); + virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const { + assert(SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() && + "invalid fpext types"); return false; } + /// Return true if an fpext operation input to an \p Opcode operation is free + /// (for instance, because half-precision floating-point numbers are + /// implicitly extended to float-precision) for an FMA instruction. + virtual bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const { + assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && + "invalid fpext types"); + return isFPExtFree(DestVT, SrcVT); + } + /// Return true if folding a vector load into ExtVal (a sign, zero, or any /// extend node) is profitable. virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const { return false; } @@ -2176,11 +2199,12 @@ public: return false; } - /// Return true if EXTRACT_SUBVECTOR is cheap for this result type - /// with this index. This is needed because EXTRACT_SUBVECTOR usually - /// has custom lowering that depends on the index of the first element, - /// and only the target knows which lowering is cheap. - virtual bool isExtractSubvectorCheap(EVT ResVT, unsigned Index) const { + /// Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type + /// from this source type with this index. This is needed because + /// EXTRACT_SUBVECTOR usually has custom lowering that depends on the index of + /// the first element, and only the target knows which lowering is cheap. + virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, + unsigned Index) const { return false; } @@ -2317,6 +2341,9 @@ private: /// backend supports. unsigned MinCmpXchgSizeInBits; + /// This indicates if the target supports unaligned atomic operations. + bool SupportsUnalignedAtomics; + /// If set to a physical register, this specifies the register that /// llvm.savestack/llvm.restorestack should save and restore. unsigned StackPointerRegisterToSaveRestore; @@ -2657,7 +2684,7 @@ public: bool AssumeSingleUse = false) const; /// Helper wrapper around SimplifyDemandedBits - bool SimplifyDemandedBits(SDValue Op, APInt &DemandedMask, + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, DAGCombinerInfo &DCI) const; /// Determine which of the bits specified in Mask are known to be either zero @@ -2670,6 +2697,15 @@ public: const SelectionDAG &DAG, unsigned Depth = 0) const; + /// Determine which of the bits of FrameIndex \p FIOp are known to be 0. + /// Default implementation computes low bits based on alignment + /// information. This should preserve known bits passed into it. + virtual void computeKnownBitsForFrameIndex(const SDValue FIOp, + KnownBits &Known, + const APInt &DemandedElts, + const SelectionDAG &DAG, + unsigned Depth = 0) const; + /// This method can be implemented by targets that want to expose additional /// information about sign bits to the DAG Combiner. The DemandedElts /// argument allows us to only collect the minimum sign bits that are shared @@ -2727,6 +2763,9 @@ public: bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const; + // For targets which wrap address, unwrap for analysis. + virtual SDValue unwrapAddress(SDValue N) const { return N; } + /// Returns true (and the GlobalValue and the offset) if the node is a /// GlobalAddress + offset. virtual bool @@ -2756,15 +2795,17 @@ public: return true; } - // Return true if it is profitable to combine a BUILD_VECTOR to a TRUNCATE. + // Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern + // to a shuffle and a truncate. // Example of such a combine: - // v4i32 build_vector((extract_elt V, 0), - // (extract_elt V, 2), - // (extract_elt V, 4), - // (extract_elt V, 6)) + // v4i32 build_vector((extract_elt V, 1), + // (extract_elt V, 3), + // (extract_elt V, 5), + // (extract_elt V, 7)) // --> - // v4i32 truncate (bitcast V to v4i64) - virtual bool isDesirableToCombineBuildVectorToTruncate() const { + // v4i32 truncate (bitcast (shuffle<1,u,3,u,5,u,7,u> V, u) to v4i64) + virtual bool isDesirableToCombineBuildVectorToShuffleTruncate( + ArrayRef<int> ShuffleMask, EVT SrcVT, EVT TruncVT) const { return false; } @@ -2866,7 +2907,7 @@ public: ArgListTy Args; SelectionDAG &DAG; SDLoc DL; - ImmutableCallSite *CS = nullptr; + ImmutableCallSite CS; SmallVector<ISD::OutputArg, 32> Outs; SmallVector<SDValue, 32> OutVals; SmallVector<ISD::InputArg, 32> Ins; @@ -2893,7 +2934,7 @@ public: RetTy = ResultType; Callee = Target; CallConv = CC; - NumFixedArgs = Args.size(); + NumFixedArgs = ArgsList.size(); Args = std::move(ArgsList); DAG.getTargetLoweringInfo().markLibCallAttributes( @@ -2906,14 +2947,14 @@ public: RetTy = ResultType; Callee = Target; CallConv = CC; - NumFixedArgs = Args.size(); + NumFixedArgs = ArgsList.size(); Args = std::move(ArgsList); return *this; } CallLoweringInfo &setCallee(Type *ResultType, FunctionType *FTy, SDValue Target, ArgListTy &&ArgsList, - ImmutableCallSite &Call) { + ImmutableCallSite Call) { RetTy = ResultType; IsInReg = Call.hasRetAttr(Attribute::InReg); @@ -2932,7 +2973,7 @@ public: NumFixedArgs = FTy->getNumParams(); Args = std::move(ArgsList); - CS = &Call; + CS = Call; return *this; } @@ -3465,6 +3506,11 @@ public: return false; } + virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, + const SDLoc &DL) const { + llvm_unreachable("not implemented for this target"); + } + /// Lower TLS global address SDNode for target independent emulated TLS model. virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const; @@ -3490,4 +3536,4 @@ void GetReturnInfo(Type *ReturnType, AttributeList attr, } // end namespace llvm -#endif // LLVM_TARGET_TARGETLOWERING_H +#endif // LLVM_CODEGEN_TARGETLOWERING_H diff --git a/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFile.h index 80d4d8e42e51..fe77c2954129 100644 --- a/contrib/llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFile.h @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetLoweringObjectFile.h - Object Info ----*- C++ -*-===// +//===-- llvm/CodeGen/TargetLoweringObjectFile.h - Object Info ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETLOWERINGOBJECTFILE_H -#define LLVM_TARGET_TARGETLOWERINGOBJECTFILE_H +#ifndef LLVM_CODEGEN_TARGETLOWERINGOBJECTFILE_H +#define LLVM_CODEGEN_TARGETLOWERINGOBJECTFILE_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" @@ -47,10 +47,10 @@ protected: bool SupportGOTPCRelWithOffset = true; /// This section contains the static constructor pointer list. - MCSection *StaticCtorSection; + MCSection *StaticCtorSection = nullptr; /// This section contains the static destructor pointer list. - MCSection *StaticDtorSection; + MCSection *StaticDtorSection = nullptr; public: TargetLoweringObjectFile() = default; @@ -191,4 +191,4 @@ protected: } // end namespace llvm -#endif // LLVM_TARGET_TARGETLOWERINGOBJECTFILE_H +#endif // LLVM_CODEGEN_TARGETLOWERINGOBJECTFILE_H diff --git a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index e4d3cc9cecfc..69de9f8cb35d 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -15,9 +15,9 @@ #ifndef LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H #define LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H +#include "llvm/CodeGen/TargetLoweringObjectFile.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCExpr.h" -#include "llvm/Target/TargetLoweringObjectFile.h" namespace llvm { @@ -182,6 +182,10 @@ public: const Function &F) const override; void InitializeWasm(); + MCSection *getStaticCtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; + MCSection *getStaticDtorSection(unsigned Priority, + const MCSymbol *KeySym) const override; const MCExpr *lowerRelativeReference(const GlobalValue *LHS, const GlobalValue *RHS, diff --git a/contrib/llvm/include/llvm/Target/TargetOpcodes.def b/contrib/llvm/include/llvm/CodeGen/TargetOpcodes.def index cadf86058f0c..d3e8483798a7 100644 --- a/contrib/llvm/include/llvm/Target/TargetOpcodes.def +++ b/contrib/llvm/include/llvm/CodeGen/TargetOpcodes.def @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetOpcodes.def - Target Indep Opcodes ------*- C++ -*-===// +//===-- llvm/CodeGen/TargetOpcodes.def - Target Indep Opcodes ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -32,6 +32,7 @@ HANDLE_TARGET_OPCODE(INLINEASM) HANDLE_TARGET_OPCODE(CFI_INSTRUCTION) HANDLE_TARGET_OPCODE(EH_LABEL) HANDLE_TARGET_OPCODE(GC_LABEL) +HANDLE_TARGET_OPCODE(ANNOTATION_LABEL) /// KILL - This instruction is a noop that is used only to adjust the /// liveness of registers. This can be useful when dealing with @@ -224,6 +225,9 @@ HANDLE_TARGET_OPCODE(G_XOR) HANDLE_TARGET_OPCODE(G_IMPLICIT_DEF) +/// Generic PHI instruction with types. +HANDLE_TARGET_OPCODE(G_PHI) + /// Generic instruction to materialize the address of an alloca or other /// stack-based object. HANDLE_TARGET_OPCODE(G_FRAME_INDEX) @@ -261,6 +265,25 @@ HANDLE_TARGET_OPCODE(G_LOAD) /// Generic store. HANDLE_TARGET_OPCODE(G_STORE) +/// Generic atomic cmpxchg with internal success check. +HANDLE_TARGET_OPCODE(G_ATOMIC_CMPXCHG_WITH_SUCCESS) + +/// Generic atomic cmpxchg. +HANDLE_TARGET_OPCODE(G_ATOMIC_CMPXCHG) + +/// Generic atomicrmw. +HANDLE_TARGET_OPCODE(G_ATOMICRMW_XCHG) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_ADD) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_SUB) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_AND) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_NAND) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_OR) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_XOR) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_MAX) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_MIN) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMAX) +HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMIN) + /// Generic conditional branch instruction. HANDLE_TARGET_OPCODE(G_BRCOND) @@ -423,12 +446,15 @@ HANDLE_TARGET_OPCODE(G_EXTRACT_VECTOR_ELT) /// Generic shufflevector. HANDLE_TARGET_OPCODE(G_SHUFFLE_VECTOR) +/// Generic byte swap. +HANDLE_TARGET_OPCODE(G_BSWAP) + // TODO: Add more generic opcodes as we move along. /// Marker for the end of the generic opcode. /// This is used to check if an opcode is in the range of the /// generic opcodes. -HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_SHUFFLE_VECTOR) +HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_BSWAP) /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific post-isel opcode values start here. diff --git a/contrib/llvm/include/llvm/Target/TargetOpcodes.h b/contrib/llvm/include/llvm/CodeGen/TargetOpcodes.h index 33df133a4d58..3ca31a970944 100644 --- a/contrib/llvm/include/llvm/Target/TargetOpcodes.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetOpcodes.h @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetOpcodes.h - Target Indep Opcodes ------*- C++ -*-===// +//===-- llvm/CodeGen/TargetOpcodes.h - Target Indep Opcodes -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETOPCODES_H -#define LLVM_TARGET_TARGETOPCODES_H +#ifndef LLVM_CODEGEN_TARGETOPCODES_H +#define LLVM_CODEGEN_TARGETOPCODES_H namespace llvm { @@ -22,19 +22,19 @@ namespace TargetOpcode { enum { #define HANDLE_TARGET_OPCODE(OPC) OPC, #define HANDLE_TARGET_OPCODE_MARKER(IDENT, OPC) IDENT = OPC, -#include "llvm/Target/TargetOpcodes.def" +#include "llvm/CodeGen/TargetOpcodes.def" }; } // end namespace TargetOpcode /// Check whether the given Opcode is a generic opcode that is not supposed /// to appear after ISel. -static inline bool isPreISelGenericOpcode(unsigned Opcode) { +inline bool isPreISelGenericOpcode(unsigned Opcode) { return Opcode >= TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START && Opcode <= TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; } /// Check whether the given Opcode is a target-specific opcode. -static inline bool isTargetSpecificOpcode(unsigned Opcode) { +inline bool isTargetSpecificOpcode(unsigned Opcode) { return Opcode > TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; } } // end namespace llvm diff --git a/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h b/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h index aaf0ab5d5481..1aaa85d77a54 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -108,6 +108,18 @@ private: bool Stopped = false; bool AddingMachinePasses = false; + /// Set the StartAfter, StartBefore and StopAfter passes to allow running only + /// a portion of the normal code-gen pass sequence. + /// + /// If the StartAfter and StartBefore pass ID is zero, then compilation will + /// begin at the normal point; otherwise, clear the Started flag to indicate + /// that passes should not be added until the starting pass is seen. If the + /// Stop pass ID is zero, then compilation will continue to the end. + /// + /// This function expects that at least one of the StartAfter or the + /// StartBefore pass IDs is null. + void setStartStopPasses(); + protected: LLVMTargetMachine *TM; PassConfigImpl *Impl = nullptr; // Internal data structures @@ -147,27 +159,25 @@ public: CodeGenOpt::Level getOptLevel() const; - /// Set the StartAfter, StartBefore and StopAfter passes to allow running only - /// a portion of the normal code-gen pass sequence. - /// - /// If the StartAfter and StartBefore pass ID is zero, then compilation will - /// begin at the normal point; otherwise, clear the Started flag to indicate - /// that passes should not be added until the starting pass is seen. If the - /// Stop pass ID is zero, then compilation will continue to the end. - /// - /// This function expects that at least one of the StartAfter or the - /// StartBefore pass IDs is null. - void setStartStopPasses(AnalysisID StartBefore, AnalysisID StartAfter, - AnalysisID StopBefore, AnalysisID StopAfter) { - assert(!(StartBefore && StartAfter) && - "Start after and start before passes are given"); - assert(!(StopBefore && StopAfter) && - "Stop after and stop before passed are given"); - this->StartBefore = StartBefore; - this->StartAfter = StartAfter; - this->StopBefore = StopBefore; - this->StopAfter = StopAfter; - Started = (StartAfter == nullptr) && (StartBefore == nullptr); + /// Describe the status of the codegen + /// pipeline set by this target pass config. + /// Having a limited codegen pipeline means that options + /// have been used to restrict what codegen is doing. + /// In particular, that means that codegen won't emit + /// assembly code. + bool hasLimitedCodeGenPipeline() const; + + /// If hasLimitedCodeGenPipeline is true, this method + /// returns a string with the name of the options, separated + /// by \p Separator that caused this pipeline to be limited. + std::string + getLimitedCodeGenPipelineReason(const char *Separator = "/") const; + + /// Check if the codegen pipeline is limited in such a way that it + /// won't be complete. When the codegen pipeline is not complete, + /// this means it may not be possible to generate assembly from it. + bool willCompleteCodeGenPipeline() const { + return !hasLimitedCodeGenPipeline() || (!StopAfter && !StopBefore); } void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); } diff --git a/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index b6839dad106f..81907538fb0b 100644 --- a/contrib/llvm/include/llvm/Target/TargetRegisterInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -1,4 +1,4 @@ -//==- Target/TargetRegisterInfo.h - Target Register Information --*- C++ -*-==// +//==- CodeGen/TargetRegisterInfo.h - Target Register Information -*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETREGISTERINFO_H -#define LLVM_TARGET_TARGETREGISTERINFO_H +#ifndef LLVM_CODEGEN_TARGETREGISTERINFO_H +#define LLVM_CODEGEN_TARGETREGISTERINFO_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" @@ -40,6 +40,7 @@ class MachineFunction; class MachineInstr; class RegScavenger; class VirtRegMap; +class LiveIntervals; class TargetRegisterClass { public: @@ -49,8 +50,6 @@ public: // Instance variables filled by tablegen, do not use! const MCRegisterClass *MC; - const uint16_t SpillSize, SpillAlignment; - const MVT::SimpleValueType *VTs; const uint32_t *SubClassMask; const uint16_t *SuperRegIndices; const LaneBitmask LaneMask; @@ -222,7 +221,10 @@ class TargetRegisterInfo : public MCRegisterInfo { public: using regclass_iterator = const TargetRegisterClass * const *; using vt_iterator = const MVT::SimpleValueType *; - + struct RegClassInfo { + unsigned RegSize, SpillSize, SpillAlignment; + vt_iterator VTList; + }; private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. @@ -231,6 +233,8 @@ private: regclass_iterator RegClassBegin, RegClassEnd; // List of regclasses LaneBitmask CoveringLanes; + const RegClassInfo *const RCInfos; + unsigned HwMode; protected: TargetRegisterInfo(const TargetRegisterInfoDesc *ID, @@ -238,7 +242,9 @@ protected: regclass_iterator RegClassEnd, const char *const *SRINames, const LaneBitmask *SRILaneMasks, - LaneBitmask CoveringLanes); + LaneBitmask CoveringLanes, + const RegClassInfo *const RSI, + unsigned Mode = 0); virtual ~TargetRegisterInfo(); public: @@ -306,37 +312,37 @@ public: /// Return the size in bits of a register from class RC. unsigned getRegSizeInBits(const TargetRegisterClass &RC) const { - return RC.SpillSize * 8; + return getRegClassInfo(RC).RegSize; } /// Return the size in bytes of the stack slot allocated to hold a spilled /// copy of a register from class RC. unsigned getSpillSize(const TargetRegisterClass &RC) const { - return RC.SpillSize; + return getRegClassInfo(RC).SpillSize / 8; } - /// Return the minimum required alignment for a spill slot for a register - /// of this class. + /// Return the minimum required alignment in bytes for a spill slot for + /// a register of this class. unsigned getSpillAlignment(const TargetRegisterClass &RC) const { - return RC.SpillAlignment; + return getRegClassInfo(RC).SpillAlignment / 8; } /// Return true if the given TargetRegisterClass has the ValueType T. bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const { - for (int i = 0; RC.VTs[i] != MVT::Other; ++i) - if (MVT(RC.VTs[i]) == T) + for (auto I = legalclasstypes_begin(RC); *I != MVT::Other; ++I) + if (MVT(*I) == T) return true; return false; } /// Loop over all of the value types that can be represented by values - // in the given register class. + /// in the given register class. vt_iterator legalclasstypes_begin(const TargetRegisterClass &RC) const { - return RC.VTs; + return getRegClassInfo(RC).VTList; } vt_iterator legalclasstypes_end(const TargetRegisterClass &RC) const { - vt_iterator I = RC.VTs; + vt_iterator I = legalclasstypes_begin(RC); while (*I != MVT::Other) ++I; return I; @@ -654,7 +660,12 @@ public: //===--------------------------------------------------------------------===// // Register Class Information // +protected: + const RegClassInfo &getRegClassInfo(const TargetRegisterClass &RC) const { + return RCInfos[getNumRegClasses() * HwMode + RC.getID()]; + } +public: /// Register class iterators regclass_iterator regclass_begin() const { return RegClassBegin; } regclass_iterator regclass_end() const { return RegClassEnd; } @@ -767,18 +778,18 @@ public: /// Get a list of 'hint' registers that the register allocator should try /// first when allocating a physical register for the virtual register /// VirtReg. These registers are effectively moved to the front of the - /// allocation order. + /// allocation order. If true is returned, regalloc will try to only use + /// hints to the greatest extent possible even if it means spilling. /// /// The Order argument is the allocation order for VirtReg's register class /// as returned from RegisterClassInfo::getOrder(). The hint registers must /// come from Order, and they must not be reserved. /// - /// The default implementation of this function can resolve - /// target-independent hints provided to MRI::setRegAllocationHint with - /// HintType == 0. Targets that override this function should defer to the - /// default implementation if they have no reason to change the allocation - /// order for VirtReg. There may be target-independent hints. - virtual void getRegAllocationHints(unsigned VirtReg, + /// The default implementation of this function will only add target + /// independent register allocation hints. Targets that override this + /// function should typically call this default implementation as well and + /// expect to see generic copy hints added. + virtual bool getRegAllocationHints(unsigned VirtReg, ArrayRef<MCPhysReg> Order, SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF, @@ -796,6 +807,13 @@ public: // Do nothing. } + /// The creation of multiple copy hints have been implemented in + /// weightCalcHelper(), but since this affects so many tests for many + /// targets, this is temporarily disabled per default. THIS SHOULD BE + /// "GENERAL GOODNESS" and hopefully all targets will update their tests + /// and enable this soon. This hook should then be removed. + virtual bool enableMultipleCopyHints() const { return false; } + /// Allow the target to reverse allocation order of local live ranges. This /// will generally allocate shorter local live ranges first. For targets with /// many registers, this could reduce regalloc compile time by a large @@ -949,7 +967,8 @@ public: unsigned SubReg, const TargetRegisterClass *DstRC, unsigned DstSubReg, - const TargetRegisterClass *NewRC) const + const TargetRegisterClass *NewRC, + LiveIntervals &LIS) const { return true; } //===--------------------------------------------------------------------===// @@ -1114,7 +1133,8 @@ public: }; // This is useful when building IndexedMaps keyed on virtual registers -struct VirtReg2IndexFunctor : public std::unary_function<unsigned, unsigned> { +struct VirtReg2IndexFunctor { + using argument_type = unsigned; unsigned operator()(unsigned Reg) const { return TargetRegisterInfo::virtReg2Index(Reg); } @@ -1124,29 +1144,34 @@ struct VirtReg2IndexFunctor : public std::unary_function<unsigned, unsigned> { /// /// The format is: /// %noreg - NoRegister -/// %vreg5 - a virtual register. -/// %vreg5:sub_8bit - a virtual register with sub-register index (with TRI). -/// %EAX - a physical register +/// %5 - a virtual register. +/// %5:sub_8bit - a virtual register with sub-register index (with TRI). +/// %eax - a physical register /// %physreg17 - a physical register when no TRI instance given. /// -/// Usage: OS << PrintReg(Reg, TRI) << '\n'; -Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI = nullptr, +/// Usage: OS << printReg(Reg, TRI, SubRegIdx) << '\n'; +Printable printReg(unsigned Reg, const TargetRegisterInfo *TRI = nullptr, unsigned SubRegIdx = 0); /// Create Printable object to print register units on a \ref raw_ostream. /// /// Register units are named after their root registers: /// -/// AL - Single root. -/// FP0~ST7 - Dual roots. +/// al - Single root. +/// fp0~st7 - Dual roots. /// -/// Usage: OS << PrintRegUnit(Unit, TRI) << '\n'; -Printable PrintRegUnit(unsigned Unit, const TargetRegisterInfo *TRI); +/// Usage: OS << printRegUnit(Unit, TRI) << '\n'; +Printable printRegUnit(unsigned Unit, const TargetRegisterInfo *TRI); /// \brief Create Printable object to print virtual registers and physical /// registers on a \ref raw_ostream. -Printable PrintVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *TRI); +Printable printVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *TRI); + +/// \brief Create Printable object to print register classes or register banks +/// on a \ref raw_ostream. +Printable printRegClassOrBank(unsigned Reg, const MachineRegisterInfo &RegInfo, + const TargetRegisterInfo *TRI); } // end namespace llvm -#endif // LLVM_TARGET_TARGETREGISTERINFO_H +#endif // LLVM_CODEGEN_TARGETREGISTERINFO_H diff --git a/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h b/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h index f23667976468..1044f0bd27e6 100644 --- a/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetSchedule.h @@ -18,9 +18,9 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/MC/MCSchedule.h" -#include "llvm/Target/TargetSubtargetInfo.h" namespace llvm { @@ -116,7 +116,7 @@ public: return SchedModel.getProcResource(PIdx); } -#ifndef NDEBUG +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) const char *getResourceName(unsigned PIdx) const { if (!PIdx) return "MOps"; diff --git a/contrib/llvm/include/llvm/Target/TargetSubtargetInfo.h b/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h index 9440c56dcf17..576522aef466 100644 --- a/contrib/llvm/include/llvm/Target/TargetSubtargetInfo.h +++ b/contrib/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -1,4 +1,4 @@ -//===- llvm/Target/TargetSubtargetInfo.h - Target Information ---*- C++ -*-===// +//===- llvm/CodeGen/TargetSubtargetInfo.h - Target Information --*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TARGET_TARGETSUBTARGETINFO_H -#define LLVM_TARGET_TARGETSUBTARGETINFO_H +#ifndef LLVM_CODEGEN_TARGETSUBTARGETINFO_H +#define LLVM_CODEGEN_TARGETSUBTARGETINFO_H #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" @@ -110,6 +110,8 @@ public: return nullptr; } + virtual unsigned getHwMode() const { return 0; } + /// Target can subclass this hook to select a different DAG scheduler. virtual RegisterScheduler::FunctionPassCtor getDAGScheduler(CodeGenOpt::Level) const { @@ -219,6 +221,11 @@ public: /// a finer grain to tune the register allocator. virtual bool enableRALocalReassignment(CodeGenOpt::Level OptLevel) const; + /// \brief True if the subtarget should consider the cost of local intervals + /// created by a split candidate when choosing the best split candidate. This + /// heuristic may be compile time intensive. + virtual bool enableAdvancedRASplitCost() const; + /// \brief Enable use of alias analysis during code generation (during MI /// scheduling, DAGCombine, etc.). virtual bool useAA() const; @@ -245,4 +252,4 @@ public: } // end namespace llvm -#endif // LLVM_TARGET_TARGETSUBTARGETINFO_H +#endif // LLVM_CODEGEN_TARGETSUBTARGETINFO_H diff --git a/contrib/llvm/include/llvm/CodeGen/ValueTypes.td b/contrib/llvm/include/llvm/CodeGen/ValueTypes.td index b1e62daa5aae..73c7fb4ce4b3 100644 --- a/contrib/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/contrib/llvm/include/llvm/CodeGen/ValueTypes.td @@ -40,110 +40,111 @@ def v8i1 : ValueType<8 , 17>; // 8 x i1 vector value def v16i1 : ValueType<16, 18>; // 16 x i1 vector value def v32i1 : ValueType<32 , 19>; // 32 x i1 vector value def v64i1 : ValueType<64 , 20>; // 64 x i1 vector value -def v512i1 : ValueType<512, 21>; // 512 x i1 vector value -def v1024i1: ValueType<1024,22>; //1024 x i1 vector value - -def v1i8 : ValueType<8, 23>; // 1 x i8 vector value -def v2i8 : ValueType<16 , 24>; // 2 x i8 vector value -def v4i8 : ValueType<32 , 25>; // 4 x i8 vector value -def v8i8 : ValueType<64 , 26>; // 8 x i8 vector value -def v16i8 : ValueType<128, 27>; // 16 x i8 vector value -def v32i8 : ValueType<256, 28>; // 32 x i8 vector value -def v64i8 : ValueType<512, 29>; // 64 x i8 vector value -def v128i8 : ValueType<1024,30>; //128 x i8 vector value -def v256i8 : ValueType<2048,31>; //256 x i8 vector value - -def v1i16 : ValueType<16 , 32>; // 1 x i16 vector value -def v2i16 : ValueType<32 , 33>; // 2 x i16 vector value -def v4i16 : ValueType<64 , 34>; // 4 x i16 vector value -def v8i16 : ValueType<128, 35>; // 8 x i16 vector value -def v16i16 : ValueType<256, 36>; // 16 x i16 vector value -def v32i16 : ValueType<512, 37>; // 32 x i16 vector value -def v64i16 : ValueType<1024,38>; // 64 x i16 vector value -def v128i16: ValueType<2048,39>; //128 x i16 vector value - -def v1i32 : ValueType<32 , 40>; // 1 x i32 vector value -def v2i32 : ValueType<64 , 41>; // 2 x i32 vector value -def v4i32 : ValueType<128, 42>; // 4 x i32 vector value -def v8i32 : ValueType<256, 43>; // 8 x i32 vector value -def v16i32 : ValueType<512, 44>; // 16 x i32 vector value -def v32i32 : ValueType<1024,45>; // 32 x i32 vector value -def v64i32 : ValueType<2048,46>; // 32 x i32 vector value - -def v1i64 : ValueType<64 , 47>; // 1 x i64 vector value -def v2i64 : ValueType<128, 48>; // 2 x i64 vector value -def v4i64 : ValueType<256, 49>; // 4 x i64 vector value -def v8i64 : ValueType<512, 50>; // 8 x i64 vector value -def v16i64 : ValueType<1024,51>; // 16 x i64 vector value -def v32i64 : ValueType<2048,52>; // 32 x i64 vector value - -def v1i128 : ValueType<128, 53>; // 1 x i128 vector value - -def nxv1i1 : ValueType<1, 54>; // n x 1 x i1 vector value -def nxv2i1 : ValueType<2, 55>; // n x 2 x i1 vector value -def nxv4i1 : ValueType<4, 56>; // n x 4 x i1 vector value -def nxv8i1 : ValueType<8, 57>; // n x 8 x i1 vector value -def nxv16i1 : ValueType<16, 58>; // n x 16 x i1 vector value -def nxv32i1 : ValueType<32, 59>; // n x 32 x i1 vector value - -def nxv1i8 : ValueType<8, 60>; // n x 1 x i8 vector value -def nxv2i8 : ValueType<16, 61>; // n x 2 x i8 vector value -def nxv4i8 : ValueType<32, 62>; // n x 4 x i8 vector value -def nxv8i8 : ValueType<64, 63>; // n x 8 x i8 vector value -def nxv16i8 : ValueType<128, 64>; // n x 16 x i8 vector value -def nxv32i8 : ValueType<256, 65>; // n x 32 x i8 vector value - -def nxv1i16 : ValueType<16, 66>; // n x 1 x i16 vector value -def nxv2i16 : ValueType<32, 67>; // n x 2 x i16 vector value -def nxv4i16 : ValueType<64, 68>; // n x 4 x i16 vector value -def nxv8i16 : ValueType<128, 69>; // n x 8 x i16 vector value -def nxv16i16: ValueType<256, 70>; // n x 16 x i16 vector value -def nxv32i16: ValueType<512, 71>; // n x 32 x i16 vector value - -def nxv1i32 : ValueType<32, 72>; // n x 1 x i32 vector value -def nxv2i32 : ValueType<64, 73>; // n x 2 x i32 vector value -def nxv4i32 : ValueType<128, 74>; // n x 4 x i32 vector value -def nxv8i32 : ValueType<256, 75>; // n x 8 x i32 vector value -def nxv16i32: ValueType<512, 76>; // n x 16 x i32 vector value -def nxv32i32: ValueType<1024,77>; // n x 32 x i32 vector value - -def nxv1i64 : ValueType<64, 78>; // n x 1 x i64 vector value -def nxv2i64 : ValueType<128, 79>; // n x 2 x i64 vector value -def nxv4i64 : ValueType<256, 80>; // n x 4 x i64 vector value -def nxv8i64 : ValueType<512, 81>; // n x 8 x i64 vector value -def nxv16i64: ValueType<1024,82>; // n x 16 x i64 vector value -def nxv32i64: ValueType<2048,83>; // n x 32 x i64 vector value - -def v2f16 : ValueType<32 , 84>; // 2 x f16 vector value -def v4f16 : ValueType<64 , 85>; // 4 x f16 vector value -def v8f16 : ValueType<128, 86>; // 8 x f16 vector value -def v1f32 : ValueType<32 , 87>; // 1 x f32 vector value -def v2f32 : ValueType<64 , 88>; // 2 x f32 vector value -def v4f32 : ValueType<128, 89>; // 4 x f32 vector value -def v8f32 : ValueType<256, 90>; // 8 x f32 vector value -def v16f32 : ValueType<512, 91>; // 16 x f32 vector value -def v1f64 : ValueType<64, 92>; // 1 x f64 vector value -def v2f64 : ValueType<128, 93>; // 2 x f64 vector value -def v4f64 : ValueType<256, 94>; // 4 x f64 vector value -def v8f64 : ValueType<512, 95>; // 8 x f64 vector value - -def nxv2f16 : ValueType<32 , 96>; // n x 2 x f16 vector value -def nxv4f16 : ValueType<64 , 97>; // n x 4 x f16 vector value -def nxv8f16 : ValueType<128, 98>; // n x 8 x f16 vector value -def nxv1f32 : ValueType<32 , 99>; // n x 1 x f32 vector value -def nxv2f32 : ValueType<64 , 100>; // n x 2 x f32 vector value -def nxv4f32 : ValueType<128, 101>; // n x 4 x f32 vector value -def nxv8f32 : ValueType<256, 102>; // n x 8 x f32 vector value -def nxv16f32 : ValueType<512, 103>; // n x 16 x f32 vector value -def nxv1f64 : ValueType<64, 104>; // n x 1 x f64 vector value -def nxv2f64 : ValueType<128, 105>; // n x 2 x f64 vector value -def nxv4f64 : ValueType<256, 106>; // n x 4 x f64 vector value -def nxv8f64 : ValueType<512, 107>; // n x 8 x f64 vector value - -def x86mmx : ValueType<64 , 108>; // X86 MMX value -def FlagVT : ValueType<0 , 109>; // Pre-RA sched glue -def isVoid : ValueType<0 , 110>; // Produces no value -def untyped: ValueType<8 , 111>; // Produces an untyped value +def v128i1 : ValueType<128, 21>; // 128 x i1 vector value +def v512i1 : ValueType<512, 22>; // 512 x i1 vector value +def v1024i1: ValueType<1024,23>; //1024 x i1 vector value + +def v1i8 : ValueType<8, 24>; // 1 x i8 vector value +def v2i8 : ValueType<16 , 25>; // 2 x i8 vector value +def v4i8 : ValueType<32 , 26>; // 4 x i8 vector value +def v8i8 : ValueType<64 , 27>; // 8 x i8 vector value +def v16i8 : ValueType<128, 28>; // 16 x i8 vector value +def v32i8 : ValueType<256, 29>; // 32 x i8 vector value +def v64i8 : ValueType<512, 30>; // 64 x i8 vector value +def v128i8 : ValueType<1024,31>; //128 x i8 vector value +def v256i8 : ValueType<2048,32>; //256 x i8 vector value + +def v1i16 : ValueType<16 , 33>; // 1 x i16 vector value +def v2i16 : ValueType<32 , 34>; // 2 x i16 vector value +def v4i16 : ValueType<64 , 35>; // 4 x i16 vector value +def v8i16 : ValueType<128, 36>; // 8 x i16 vector value +def v16i16 : ValueType<256, 37>; // 16 x i16 vector value +def v32i16 : ValueType<512, 38>; // 32 x i16 vector value +def v64i16 : ValueType<1024,39>; // 64 x i16 vector value +def v128i16: ValueType<2048,40>; //128 x i16 vector value + +def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value +def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value +def v4i32 : ValueType<128, 43>; // 4 x i32 vector value +def v8i32 : ValueType<256, 44>; // 8 x i32 vector value +def v16i32 : ValueType<512, 45>; // 16 x i32 vector value +def v32i32 : ValueType<1024,46>; // 32 x i32 vector value +def v64i32 : ValueType<2048,47>; // 32 x i32 vector value + +def v1i64 : ValueType<64 , 48>; // 1 x i64 vector value +def v2i64 : ValueType<128, 49>; // 2 x i64 vector value +def v4i64 : ValueType<256, 50>; // 4 x i64 vector value +def v8i64 : ValueType<512, 51>; // 8 x i64 vector value +def v16i64 : ValueType<1024,52>; // 16 x i64 vector value +def v32i64 : ValueType<2048,53>; // 32 x i64 vector value + +def v1i128 : ValueType<128, 54>; // 1 x i128 vector value + +def nxv1i1 : ValueType<1, 55>; // n x 1 x i1 vector value +def nxv2i1 : ValueType<2, 56>; // n x 2 x i1 vector value +def nxv4i1 : ValueType<4, 57>; // n x 4 x i1 vector value +def nxv8i1 : ValueType<8, 58>; // n x 8 x i1 vector value +def nxv16i1 : ValueType<16, 59>; // n x 16 x i1 vector value +def nxv32i1 : ValueType<32, 60>; // n x 32 x i1 vector value + +def nxv1i8 : ValueType<8, 61>; // n x 1 x i8 vector value +def nxv2i8 : ValueType<16, 62>; // n x 2 x i8 vector value +def nxv4i8 : ValueType<32, 63>; // n x 4 x i8 vector value +def nxv8i8 : ValueType<64, 64>; // n x 8 x i8 vector value +def nxv16i8 : ValueType<128, 65>; // n x 16 x i8 vector value +def nxv32i8 : ValueType<256, 66>; // n x 32 x i8 vector value + +def nxv1i16 : ValueType<16, 67>; // n x 1 x i16 vector value +def nxv2i16 : ValueType<32, 68>; // n x 2 x i16 vector value +def nxv4i16 : ValueType<64, 69>; // n x 4 x i16 vector value +def nxv8i16 : ValueType<128, 70>; // n x 8 x i16 vector value +def nxv16i16: ValueType<256, 71>; // n x 16 x i16 vector value +def nxv32i16: ValueType<512, 72>; // n x 32 x i16 vector value + +def nxv1i32 : ValueType<32, 73>; // n x 1 x i32 vector value +def nxv2i32 : ValueType<64, 74>; // n x 2 x i32 vector value +def nxv4i32 : ValueType<128, 75>; // n x 4 x i32 vector value +def nxv8i32 : ValueType<256, 76>; // n x 8 x i32 vector value +def nxv16i32: ValueType<512, 77>; // n x 16 x i32 vector value +def nxv32i32: ValueType<1024,78>; // n x 32 x i32 vector value + +def nxv1i64 : ValueType<64, 79>; // n x 1 x i64 vector value +def nxv2i64 : ValueType<128, 80>; // n x 2 x i64 vector value +def nxv4i64 : ValueType<256, 81>; // n x 4 x i64 vector value +def nxv8i64 : ValueType<512, 82>; // n x 8 x i64 vector value +def nxv16i64: ValueType<1024,83>; // n x 16 x i64 vector value +def nxv32i64: ValueType<2048,84>; // n x 32 x i64 vector value + +def v2f16 : ValueType<32 , 85>; // 2 x f16 vector value +def v4f16 : ValueType<64 , 86>; // 4 x f16 vector value +def v8f16 : ValueType<128, 87>; // 8 x f16 vector value +def v1f32 : ValueType<32 , 88>; // 1 x f32 vector value +def v2f32 : ValueType<64 , 89>; // 2 x f32 vector value +def v4f32 : ValueType<128, 90>; // 4 x f32 vector value +def v8f32 : ValueType<256, 91>; // 8 x f32 vector value +def v16f32 : ValueType<512, 92>; // 16 x f32 vector value +def v1f64 : ValueType<64, 93>; // 1 x f64 vector value +def v2f64 : ValueType<128, 94>; // 2 x f64 vector value +def v4f64 : ValueType<256, 95>; // 4 x f64 vector value +def v8f64 : ValueType<512, 96>; // 8 x f64 vector value + +def nxv2f16 : ValueType<32 , 97>; // n x 2 x f16 vector value +def nxv4f16 : ValueType<64 , 98>; // n x 4 x f16 vector value +def nxv8f16 : ValueType<128, 99>; // n x 8 x f16 vector value +def nxv1f32 : ValueType<32 , 100>; // n x 1 x f32 vector value +def nxv2f32 : ValueType<64 , 101>; // n x 2 x f32 vector value +def nxv4f32 : ValueType<128, 102>; // n x 4 x f32 vector value +def nxv8f32 : ValueType<256, 103>; // n x 8 x f32 vector value +def nxv16f32 : ValueType<512, 104>; // n x 16 x f32 vector value +def nxv1f64 : ValueType<64, 105>; // n x 1 x f64 vector value +def nxv2f64 : ValueType<128, 106>; // n x 2 x f64 vector value +def nxv4f64 : ValueType<256, 107>; // n x 4 x f64 vector value +def nxv8f64 : ValueType<512, 108>; // n x 8 x f64 vector value + +def x86mmx : ValueType<64 , 109>; // X86 MMX value +def FlagVT : ValueType<0 , 110>; // Pre-RA sched glue +def isVoid : ValueType<0 , 111>; // Produces no value +def untyped: ValueType<8 , 112>; // Produces an untyped value def token : ValueType<0 , 248>; // TokenTy def MetadataVT: ValueType<0, 249>; // Metadata diff --git a/contrib/llvm/include/llvm/CodeGen/VirtRegMap.h b/contrib/llvm/include/llvm/CodeGen/VirtRegMap.h index b9076353fd07..3b06f0393114 100644 --- a/contrib/llvm/include/llvm/CodeGen/VirtRegMap.h +++ b/contrib/llvm/include/llvm/CodeGen/VirtRegMap.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/VirtRegMap.h - Virtual Register Map -*- C++ -*--------===// +//===- llvm/CodeGen/VirtRegMap.h - Virtual Register Map ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -19,15 +19,17 @@ #include "llvm/ADT/IndexedMap.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Pass.h" +#include <cassert> namespace llvm { - class MachineInstr; - class MachineFunction; - class MachineRegisterInfo; - class TargetInstrInfo; - class raw_ostream; - class SlotIndexes; + +class MachineFunction; +class MachineRegisterInfo; +class raw_ostream; +class TargetInstrInfo; class VirtRegMap : public MachineFunctionPass { public: @@ -63,13 +65,14 @@ namespace llvm { /// createSpillSlot - Allocate a spill slot for RC from MFI. unsigned createSpillSlot(const TargetRegisterClass *RC); - VirtRegMap(const VirtRegMap&) = delete; - void operator=(const VirtRegMap&) = delete; - public: static char ID; + VirtRegMap() : MachineFunctionPass(ID), Virt2PhysMap(NO_PHYS_REG), - Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) { } + Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) {} + VirtRegMap(const VirtRegMap &) = delete; + VirtRegMap &operator=(const VirtRegMap &) = delete; + bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -166,6 +169,7 @@ namespace llvm { /// @brief create a mapping for the specifed virtual register to /// the next available stack slot int assignVirt2StackSlot(unsigned virtReg); + /// @brief create a mapping for the specified virtual register to /// the specified stack slot void assignVirt2StackSlot(unsigned virtReg, int frameIndex); @@ -178,6 +182,7 @@ namespace llvm { VRM.print(OS); return OS; } -} // End llvm namespace -#endif +} // end llvm namespace + +#endif // LLVM_CODEGEN_VIRTREGMAP_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h new file mode 100644 index 000000000000..bd1743511ed4 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h @@ -0,0 +1,70 @@ +//===- AppendingTypeTableBuilder.h -------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_APPENDINGTYPETABLEBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_APPENDINGTYPETABLEBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" +#include "llvm/DebugInfo/CodeView/TypeCollection.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/Support/Allocator.h" +#include <cassert> +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { +namespace codeview { + +class ContinuationRecordBuilder; + +class AppendingTypeTableBuilder : public TypeCollection { + + BumpPtrAllocator &RecordStorage; + SimpleTypeSerializer SimpleSerializer; + + /// Contains a list of all records indexed by TypeIndex.toArrayIndex(). + SmallVector<ArrayRef<uint8_t>, 2> SeenRecords; + +public: + explicit AppendingTypeTableBuilder(BumpPtrAllocator &Storage); + ~AppendingTypeTableBuilder(); + + // TypeTableCollection overrides + Optional<TypeIndex> getFirst() override; + Optional<TypeIndex> getNext(TypeIndex Prev) override; + CVType getType(TypeIndex Index) override; + StringRef getTypeName(TypeIndex Index) override; + bool contains(TypeIndex Index) override; + uint32_t size() override; + uint32_t capacity() override; + + // public interface + void reset(); + TypeIndex nextTypeIndex() const; + + BumpPtrAllocator &getAllocator() { return RecordStorage; } + + ArrayRef<ArrayRef<uint8_t>> records() const; + TypeIndex insertRecordBytes(ArrayRef<uint8_t> &Record); + TypeIndex insertRecord(ContinuationRecordBuilder &Builder); + + template <typename T> TypeIndex writeLeafType(T &Record) { + ArrayRef<uint8_t> Data = SimpleSerializer.serialize(Record); + return insertRecordBytes(Data); + } +}; + +} // end namespace codeview +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h index 44040e04388a..9f3a753ad1ae 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CVRecord.h @@ -61,30 +61,38 @@ template <typename Kind> struct RemappedRecord { SmallVector<std::pair<uint32_t, TypeIndex>, 8> Mappings; }; +/// Read a complete record from a stream at a random offset. +template <typename Kind> +inline Expected<CVRecord<Kind>> readCVRecordFromStream(BinaryStreamRef Stream, + uint32_t Offset) { + const RecordPrefix *Prefix = nullptr; + BinaryStreamReader Reader(Stream); + Reader.setOffset(Offset); + + if (auto EC = Reader.readObject(Prefix)) + return std::move(EC); + if (Prefix->RecordLen < 2) + return make_error<CodeViewError>(cv_error_code::corrupt_record); + Kind K = static_cast<Kind>(uint16_t(Prefix->RecordKind)); + + Reader.setOffset(Offset); + ArrayRef<uint8_t> RawData; + if (auto EC = Reader.readBytes(RawData, Prefix->RecordLen + sizeof(uint16_t))) + return std::move(EC); + return codeview::CVRecord<Kind>(K, RawData); +} + } // end namespace codeview template <typename Kind> struct VarStreamArrayExtractor<codeview::CVRecord<Kind>> { Error operator()(BinaryStreamRef Stream, uint32_t &Len, codeview::CVRecord<Kind> &Item) { - using namespace codeview; - const RecordPrefix *Prefix = nullptr; - BinaryStreamReader Reader(Stream); - uint32_t Offset = Reader.getOffset(); - - if (auto EC = Reader.readObject(Prefix)) - return EC; - if (Prefix->RecordLen < 2) - return make_error<CodeViewError>(cv_error_code::corrupt_record); - Kind K = static_cast<Kind>(uint16_t(Prefix->RecordKind)); - - Reader.setOffset(Offset); - ArrayRef<uint8_t> RawData; - if (auto EC = - Reader.readBytes(RawData, Prefix->RecordLen + sizeof(uint16_t))) - return EC; - Item = codeview::CVRecord<Kind>(K, RawData); - Len = Item.length(); + auto ExpectedRec = codeview::readCVRecordFromStream<Kind>(Stream, 0); + if (!ExpectedRec) + return ExpectedRec.takeError(); + Item = *ExpectedRec; + Len = ExpectedRec->length(); return Error::success(); } }; diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h index b7a7e33abadf..1a4f510c24ab 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeView.h @@ -124,6 +124,7 @@ enum class CPUType : uint16_t { ARM_XMAC = 0x66, ARM_WMMX = 0x67, ARM7 = 0x68, + ARM64 = 0x69, Omni = 0x70, Ia64 = 0x80, Ia64_2 = 0x81, @@ -157,7 +158,11 @@ enum SourceLanguage : uint8_t { Java = 0x0d, JScript = 0x0e, MSIL = 0x0f, - HLSL = 0x10 + HLSL = 0x10, + + /// The DMD compiler emits 'D' for the CV source language. Microsoft doesn't + /// have an enumerator for it yet. + D = 'D', }; /// These values correspond to the CV_call_e enumeration, and are documented @@ -500,55 +505,9 @@ enum class FrameCookieKind : uint8_t { // Corresponds to CV_HREG_e enum. enum class RegisterId : uint16_t { - Unknown = 0, - VFrame = 30006, - AL = 1, - CL = 2, - DL = 3, - BL = 4, - AH = 5, - CH = 6, - DH = 7, - BH = 8, - AX = 9, - CX = 10, - DX = 11, - BX = 12, - SP = 13, - BP = 14, - SI = 15, - DI = 16, - EAX = 17, - ECX = 18, - EDX = 19, - EBX = 20, - ESP = 21, - EBP = 22, - ESI = 23, - EDI = 24, - ES = 25, - CS = 26, - SS = 27, - DS = 28, - FS = 29, - GS = 30, - IP = 31, - RAX = 328, - RBX = 329, - RCX = 330, - RDX = 331, - RSI = 332, - RDI = 333, - RBP = 334, - RSP = 335, - R8 = 336, - R9 = 337, - R10 = 338, - R11 = 339, - R12 = 340, - R13 = 341, - R14 = 342, - R15 = 343, +#define CV_REGISTER(name, value) name = value, +#include "CodeViewRegisters.def" +#undef CV_REGISTER }; /// These values correspond to the THUNK_ORDINAL enumeration. diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def new file mode 100644 index 000000000000..3f0660294866 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def @@ -0,0 +1,268 @@ +//===-- CodeViewRegisters.def - CodeView registers --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// See CV_HREG_e in cvconst.h. This should match the constants there. +// +//===----------------------------------------------------------------------===// + +#ifndef CV_REGISTER +#define CV_REGISTER(name, value) +#endif + +// This currently only contains the "register subset shraed by all processor +// types" (ERR etc.) and the x86 registers. + +CV_REGISTER(ERR, 30000) +CV_REGISTER(TEB, 30001) +CV_REGISTER(TIMER, 30002) +CV_REGISTER(EFAD1, 30003) +CV_REGISTER(EFAD2, 30004) +CV_REGISTER(EFAD3, 30005) +CV_REGISTER(VFRAME, 30006) +CV_REGISTER(HANDLE, 30007) +CV_REGISTER(PARAMS, 30008) +CV_REGISTER(LOCALS, 30009) +CV_REGISTER(TID, 30010) +CV_REGISTER(ENV, 30011) +CV_REGISTER(CMDLN, 30012) + +CV_REGISTER(NONE, 0) +CV_REGISTER(AL, 1) +CV_REGISTER(CL, 2) +CV_REGISTER(DL, 3) +CV_REGISTER(BL, 4) +CV_REGISTER(AH, 5) +CV_REGISTER(CH, 6) +CV_REGISTER(DH, 7) +CV_REGISTER(BH, 8) +CV_REGISTER(AX, 9) +CV_REGISTER(CX, 10) +CV_REGISTER(DX, 11) +CV_REGISTER(BX, 12) +CV_REGISTER(SP, 13) +CV_REGISTER(BP, 14) +CV_REGISTER(SI, 15) +CV_REGISTER(DI, 16) +CV_REGISTER(EAX, 17) +CV_REGISTER(ECX, 18) +CV_REGISTER(EDX, 19) +CV_REGISTER(EBX, 20) +CV_REGISTER(ESP, 21) +CV_REGISTER(EBP, 22) +CV_REGISTER(ESI, 23) +CV_REGISTER(EDI, 24) +CV_REGISTER(ES, 25) +CV_REGISTER(CS, 26) +CV_REGISTER(SS, 27) +CV_REGISTER(DS, 28) +CV_REGISTER(FS, 29) +CV_REGISTER(GS, 30) +CV_REGISTER(IP, 31) +CV_REGISTER(FLAGS, 32) +CV_REGISTER(EIP, 33) +CV_REGISTER(EFLAGS, 34) +CV_REGISTER(TEMP, 40) +CV_REGISTER(TEMPH, 41) +CV_REGISTER(QUOTE, 42) +CV_REGISTER(PCDR3, 43) +CV_REGISTER(PCDR4, 44) +CV_REGISTER(PCDR5, 45) +CV_REGISTER(PCDR6, 46) +CV_REGISTER(PCDR7, 47) +CV_REGISTER(CR0, 80) +CV_REGISTER(CR1, 81) +CV_REGISTER(CR2, 82) +CV_REGISTER(CR3, 83) +CV_REGISTER(CR4, 84) +CV_REGISTER(DR0, 90) +CV_REGISTER(DR1, 91) +CV_REGISTER(DR2, 92) +CV_REGISTER(DR3, 93) +CV_REGISTER(DR4, 94) +CV_REGISTER(DR5, 95) +CV_REGISTER(DR6, 96) +CV_REGISTER(DR7, 97) +CV_REGISTER(GDTR, 110) +CV_REGISTER(GDTL, 111) +CV_REGISTER(IDTR, 112) +CV_REGISTER(IDTL, 113) +CV_REGISTER(LDTR, 114) +CV_REGISTER(TR, 115) + +CV_REGISTER(PSEUDO1, 116) +CV_REGISTER(PSEUDO2, 117) +CV_REGISTER(PSEUDO3, 118) +CV_REGISTER(PSEUDO4, 119) +CV_REGISTER(PSEUDO5, 120) +CV_REGISTER(PSEUDO6, 121) +CV_REGISTER(PSEUDO7, 122) +CV_REGISTER(PSEUDO8, 123) +CV_REGISTER(PSEUDO9, 124) + +CV_REGISTER(ST0, 128) +CV_REGISTER(ST1, 129) +CV_REGISTER(ST2, 130) +CV_REGISTER(ST3, 131) +CV_REGISTER(ST4, 132) +CV_REGISTER(ST5, 133) +CV_REGISTER(ST6, 134) +CV_REGISTER(ST7, 135) +CV_REGISTER(CTRL, 136) +CV_REGISTER(STAT, 137) +CV_REGISTER(TAG, 138) +CV_REGISTER(FPIP, 139) +CV_REGISTER(FPCS, 140) +CV_REGISTER(FPDO, 141) +CV_REGISTER(FPDS, 142) +CV_REGISTER(ISEM, 143) +CV_REGISTER(FPEIP, 144) +CV_REGISTER(FPEDO, 145) + +CV_REGISTER(MM0, 146) +CV_REGISTER(MM1, 147) +CV_REGISTER(MM2, 148) +CV_REGISTER(MM3, 149) +CV_REGISTER(MM4, 150) +CV_REGISTER(MM5, 151) +CV_REGISTER(MM6, 152) +CV_REGISTER(MM7, 153) + +CV_REGISTER(XMM0, 154) +CV_REGISTER(XMM1, 155) +CV_REGISTER(XMM2, 156) +CV_REGISTER(XMM3, 157) +CV_REGISTER(XMM4, 158) +CV_REGISTER(XMM5, 159) +CV_REGISTER(XMM6, 160) +CV_REGISTER(XMM7, 161) + +CV_REGISTER(MXCSR, 211) + +CV_REGISTER(EDXEAX, 212) + +CV_REGISTER(EMM0L, 220) +CV_REGISTER(EMM1L, 221) +CV_REGISTER(EMM2L, 222) +CV_REGISTER(EMM3L, 223) +CV_REGISTER(EMM4L, 224) +CV_REGISTER(EMM5L, 225) +CV_REGISTER(EMM6L, 226) +CV_REGISTER(EMM7L, 227) + +CV_REGISTER(EMM0H, 228) +CV_REGISTER(EMM1H, 229) +CV_REGISTER(EMM2H, 230) +CV_REGISTER(EMM3H, 231) +CV_REGISTER(EMM4H, 232) +CV_REGISTER(EMM5H, 233) +CV_REGISTER(EMM6H, 234) +CV_REGISTER(EMM7H, 235) + +CV_REGISTER(MM00, 236) +CV_REGISTER(MM01, 237) +CV_REGISTER(MM10, 238) +CV_REGISTER(MM11, 239) +CV_REGISTER(MM20, 240) +CV_REGISTER(MM21, 241) +CV_REGISTER(MM30, 242) +CV_REGISTER(MM31, 243) +CV_REGISTER(MM40, 244) +CV_REGISTER(MM41, 245) +CV_REGISTER(MM50, 246) +CV_REGISTER(MM51, 247) +CV_REGISTER(MM60, 248) +CV_REGISTER(MM61, 249) +CV_REGISTER(MM70, 250) +CV_REGISTER(MM71, 251) + +CV_REGISTER(BND0, 396) +CV_REGISTER(BND1, 397) +CV_REGISTER(BND2, 398) + + +CV_REGISTER(XMM8, 252) +CV_REGISTER(XMM9, 253) +CV_REGISTER(XMM10, 254) +CV_REGISTER(XMM11, 255) +CV_REGISTER(XMM12, 256) +CV_REGISTER(XMM13, 257) +CV_REGISTER(XMM14, 258) +CV_REGISTER(XMM15, 259) + + +CV_REGISTER(SIL, 324) +CV_REGISTER(DIL, 325) +CV_REGISTER(BPL, 326) +CV_REGISTER(SPL, 327) + +CV_REGISTER(RAX, 328) +CV_REGISTER(RBX, 329) +CV_REGISTER(RCX, 330) +CV_REGISTER(RDX, 331) +CV_REGISTER(RSI, 332) +CV_REGISTER(RDI, 333) +CV_REGISTER(RBP, 334) +CV_REGISTER(RSP, 335) + +CV_REGISTER(R8, 336) +CV_REGISTER(R9, 337) +CV_REGISTER(R10, 338) +CV_REGISTER(R11, 339) +CV_REGISTER(R12, 340) +CV_REGISTER(R13, 341) +CV_REGISTER(R14, 342) +CV_REGISTER(R15, 343) + +CV_REGISTER(R8B, 344) +CV_REGISTER(R9B, 345) +CV_REGISTER(R10B, 346) +CV_REGISTER(R11B, 347) +CV_REGISTER(R12B, 348) +CV_REGISTER(R13B, 349) +CV_REGISTER(R14B, 350) +CV_REGISTER(R15B, 351) + +CV_REGISTER(R8W, 352) +CV_REGISTER(R9W, 353) +CV_REGISTER(R10W, 354) +CV_REGISTER(R11W, 355) +CV_REGISTER(R12W, 356) +CV_REGISTER(R13W, 357) +CV_REGISTER(R14W, 358) +CV_REGISTER(R15W, 359) + +CV_REGISTER(R8D, 360) +CV_REGISTER(R9D, 361) +CV_REGISTER(R10D, 362) +CV_REGISTER(R11D, 363) +CV_REGISTER(R12D, 364) +CV_REGISTER(R13D, 365) +CV_REGISTER(R14D, 366) +CV_REGISTER(R15D, 367) + + +// cvconst.h defines both CV_REG_YMM0 (252) and CV_AMD64_YMM0 (368). Keep the +// original prefix to distinguish them. + +CV_REGISTER(AMD64_YMM0, 368) +CV_REGISTER(AMD64_YMM1, 369) +CV_REGISTER(AMD64_YMM2, 370) +CV_REGISTER(AMD64_YMM3, 371) +CV_REGISTER(AMD64_YMM4, 372) +CV_REGISTER(AMD64_YMM5, 373) +CV_REGISTER(AMD64_YMM6, 374) +CV_REGISTER(AMD64_YMM7, 375) +CV_REGISTER(AMD64_YMM8, 376) +CV_REGISTER(AMD64_YMM9, 377) +CV_REGISTER(AMD64_YMM10, 378) +CV_REGISTER(AMD64_YMM11, 379) +CV_REGISTER(AMD64_YMM12, 380) +CV_REGISTER(AMD64_YMM13, 381) +CV_REGISTER(AMD64_YMM14, 382) +CV_REGISTER(AMD64_YMM15, 383) diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def index 32813d861d90..41c538076798 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def @@ -1,4 +1,4 @@ -//===-- CVLeafTypes.def - All CodeView leaf types ---------------*- C++ -*-===// +//===-- CodeViewSymbols.def - All CodeView leaf types -----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -184,6 +184,9 @@ CV_SYMBOL(S_LDATA_HLSL32 , 0x1163) CV_SYMBOL(S_GDATA_HLSL32_EX, 0x1164) CV_SYMBOL(S_LDATA_HLSL32_EX, 0x1165) +CV_SYMBOL(S_FASTLINK, 0x1167) // Undocumented +SYMBOL_RECORD_ALIAS(S_INLINEES, 0x1168, InlineesSym, CallerSym) // Undocumented + // Known symbol types SYMBOL_RECORD(S_END , 0x0006, ScopeEndSym) SYMBOL_RECORD_ALIAS(S_INLINESITE_END , 0x114e, InlineSiteEnd, ScopeEndSym) @@ -232,7 +235,7 @@ SYMBOL_RECORD(S_HEAPALLOCSITE , 0x115e, HeapAllocationSiteSym) SYMBOL_RECORD(S_FRAMECOOKIE , 0x113a, FrameCookieSym) SYMBOL_RECORD(S_CALLEES , 0x115a, CallerSym) -SYMBOL_RECORD_ALIAS(S_CALLERS , 0x115b, CalleeSym, CallerSym) +SYMBOL_RECORD_ALIAS(S_CALLERS, 0x115b, CalleeSym, CallerSym) SYMBOL_RECORD(S_UDT , 0x1108, UDTSym) SYMBOL_RECORD_ALIAS(S_COBOLUDT , 0x1109, CobolUDT, UDTSym) diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def index 8c193bb13cb7..69ce9606a670 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def @@ -1,5 +1,4 @@ - -//===-- CVLeafTypes.def - All CodeView leaf types ---------------*- C++ -*-===// +//===-- CodeViewTypes.def - All CodeView leaf types -------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h new file mode 100644 index 000000000000..7f851a2595dc --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h @@ -0,0 +1,65 @@ +//===- ContinuationRecordBuilder.h ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_CONTINUATIONRECORDBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_CONTINUATIONRECORDBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/RecordSerialization.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeRecordMapping.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/Error.h" +#include <cassert> +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { +namespace codeview { +enum class ContinuationRecordKind { FieldList, MethodOverloadList }; + +class ContinuationRecordBuilder { + SmallVector<uint32_t, 4> SegmentOffsets; + Optional<ContinuationRecordKind> Kind; + AppendingBinaryByteStream Buffer; + BinaryStreamWriter SegmentWriter; + TypeRecordMapping Mapping; + ArrayRef<uint8_t> InjectedSegmentBytes; + + uint32_t getCurrentSegmentLength() const; + + void insertSegmentEnd(uint32_t Offset); + CVType createSegmentRecord(uint32_t OffBegin, uint32_t OffEnd, + Optional<TypeIndex> RefersTo); + +public: + ContinuationRecordBuilder(); + ~ContinuationRecordBuilder(); + + void begin(ContinuationRecordKind RecordKind); + + // This template is explicitly instantiated in the implementation file for all + // supported types. The method itself is ugly, so inlining it into the header + // file clutters an otherwise straightforward interface. + template <typename RecordType> void writeMemberType(RecordType &Record); + + std::vector<CVType> end(TypeIndex Index); +}; +} // namespace codeview +} // namespace llvm + +#endif
\ No newline at end of file diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h new file mode 100644 index 000000000000..d8ac3343c15f --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -0,0 +1,87 @@ +//===- GlobalTypeTableBuilder.h ----------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_GLOBALTYPETABLEBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_GLOBALTYPETABLEBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" +#include "llvm/DebugInfo/CodeView/TypeCollection.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/Support/Allocator.h" +#include <cassert> +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { +namespace codeview { + +class ContinuationRecordBuilder; + +class GlobalTypeTableBuilder : public TypeCollection { + /// Storage for records. These need to outlive the TypeTableBuilder. + BumpPtrAllocator &RecordStorage; + + /// A serializer that can write non-continuation leaf types. Only used as + /// a convenience function so that we can provide an interface method to + /// write an unserialized record. + SimpleTypeSerializer SimpleSerializer; + + /// Hash table. + DenseMap<GloballyHashedType, TypeIndex> HashedRecords; + + /// Contains a list of all records indexed by TypeIndex.toArrayIndex(). + SmallVector<ArrayRef<uint8_t>, 2> SeenRecords; + + /// Contains a list of all hash values inexed by TypeIndex.toArrayIndex(). + SmallVector<GloballyHashedType, 2> SeenHashes; + +public: + explicit GlobalTypeTableBuilder(BumpPtrAllocator &Storage); + ~GlobalTypeTableBuilder(); + + // TypeTableCollection overrides + Optional<TypeIndex> getFirst() override; + Optional<TypeIndex> getNext(TypeIndex Prev) override; + CVType getType(TypeIndex Index) override; + StringRef getTypeName(TypeIndex Index) override; + bool contains(TypeIndex Index) override; + uint32_t size() override; + uint32_t capacity() override; + + // public interface + void reset(); + TypeIndex nextTypeIndex() const; + + BumpPtrAllocator &getAllocator() { return RecordStorage; } + + ArrayRef<ArrayRef<uint8_t>> records() const; + ArrayRef<GloballyHashedType> hashes() const; + + using CreateRecord = llvm::function_ref<ArrayRef<uint8_t>()>; + + TypeIndex insertRecordAs(GloballyHashedType Hash, CreateRecord Create); + TypeIndex insertRecordBytes(ArrayRef<uint8_t> Data); + TypeIndex insertRecord(ContinuationRecordBuilder &Builder); + + template <typename T> TypeIndex writeLeafType(T &Record) { + ArrayRef<uint8_t> Data = SimpleSerializer.serialize(Record); + return insertRecordBytes(Data); + } +}; + +} // end namespace codeview +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_MERGINGTYPETABLEBUILDER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h index cc0c24301d49..16d78692c839 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h @@ -67,9 +67,12 @@ public: void reset(ArrayRef<uint8_t> Data, uint32_t RecordCountHint); void reset(StringRef Data, uint32_t RecordCountHint); + void reset(BinaryStreamReader &Reader, uint32_t RecordCountHint); uint32_t getOffsetOfType(TypeIndex Index); + Optional<CVType> tryGetType(TypeIndex Index); + CVType getType(TypeIndex Index) override; StringRef getTypeName(TypeIndex Index) override; bool contains(TypeIndex Index) override; diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h new file mode 100644 index 000000000000..9030918ebbb3 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h @@ -0,0 +1,81 @@ +//===- MergingTypeTableBuilder.h ---------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_MERGINGTYPETABLEBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_MERGINGTYPETABLEBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/SimpleTypeSerializer.h" +#include "llvm/DebugInfo/CodeView/TypeCollection.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/Support/Allocator.h" +#include <cassert> +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { +namespace codeview { + +class ContinuationRecordBuilder; + +class MergingTypeTableBuilder : public TypeCollection { + /// Storage for records. These need to outlive the TypeTableBuilder. + BumpPtrAllocator &RecordStorage; + + /// A serializer that can write non-continuation leaf types. Only used as + /// a convenience function so that we can provide an interface method to + /// write an unserialized record. + SimpleTypeSerializer SimpleSerializer; + + /// Hash table. + DenseMap<LocallyHashedType, TypeIndex> HashedRecords; + + /// Contains a list of all records indexed by TypeIndex.toArrayIndex(). + SmallVector<ArrayRef<uint8_t>, 2> SeenRecords; + +public: + explicit MergingTypeTableBuilder(BumpPtrAllocator &Storage); + ~MergingTypeTableBuilder(); + + // TypeTableCollection overrides + Optional<TypeIndex> getFirst() override; + Optional<TypeIndex> getNext(TypeIndex Prev) override; + CVType getType(TypeIndex Index) override; + StringRef getTypeName(TypeIndex Index) override; + bool contains(TypeIndex Index) override; + uint32_t size() override; + uint32_t capacity() override; + + // public interface + void reset(); + TypeIndex nextTypeIndex() const; + + BumpPtrAllocator &getAllocator() { return RecordStorage; } + + ArrayRef<ArrayRef<uint8_t>> records() const; + + TypeIndex insertRecordAs(hash_code Hash, ArrayRef<uint8_t> &Record); + TypeIndex insertRecordBytes(ArrayRef<uint8_t> &Record); + TypeIndex insertRecord(ContinuationRecordBuilder &Builder); + + template <typename T> TypeIndex writeLeafType(T &Record) { + ArrayRef<uint8_t> Data = SimpleSerializer.serialize(Record); + return insertRecordBytes(Data); + } +}; + +} // end namespace codeview +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_MERGINGTYPETABLEBUILDER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeName.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordName.h index a987b4afd283..b022108df3d6 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeName.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/RecordName.h @@ -1,4 +1,4 @@ -//===- TypeName.h --------------------------------------------- *- C++ --*-===// +//===- RecordName.h ------------------------------------------- *- C++ --*-===// // // The LLVM Compiler Infrastructure // @@ -7,16 +7,18 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPENAME_H -#define LLVM_DEBUGINFO_CODEVIEW_TYPENAME_H +#ifndef LLVM_DEBUGINFO_CODEVIEW_RECORDNAME_H +#define LLVM_DEBUGINFO_CODEVIEW_RECORDNAME_H +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/TypeCollection.h" #include "llvm/DebugInfo/CodeView/TypeIndex.h" namespace llvm { namespace codeview { std::string computeTypeName(TypeCollection &Types, TypeIndex Index); -} +StringRef getSymbolName(CVSymbol Sym); +} // namespace codeview } // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h new file mode 100644 index 000000000000..a85d9270186b --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h @@ -0,0 +1,53 @@ +//===- SimpleTypeSerializer.h -----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_SIMPLETYPESERIALIZER_H +#define LLVM_DEBUGINFO_CODEVIEW_SIMPLETYPESERIALIZER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/RecordSerialization.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeRecordMapping.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/Error.h" +#include <cassert> +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { +namespace codeview { + +class SimpleTypeSerializer { + std::vector<uint8_t> ScratchBuffer; + +public: + SimpleTypeSerializer(); + ~SimpleTypeSerializer(); + + // This template is explicitly instantiated in the implementation file for all + // supported types. The method itself is ugly, so inlining it into the header + // file clutters an otherwise straightforward interface. + template <typename T> ArrayRef<uint8_t> serialize(T &Record); + + // Don't allow serialization of field list records using this interface. + ArrayRef<uint8_t> serialize(const FieldListRecord &Record) = delete; +}; + +} // end namespace codeview +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_SIMPLETYPESERIALIZER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h index 1a8388224665..22a333e631a0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h @@ -31,8 +31,13 @@ public: StringsAndChecksumsRef(const DebugStringTableSubsectionRef &Strings, const DebugChecksumsSubsectionRef &Checksums); + void setStrings(const DebugStringTableSubsectionRef &Strings); void setChecksums(const DebugChecksumsSubsectionRef &CS); + void reset(); + void resetStrings(); + void resetChecksums(); + template <typename T> void initialize(T &&FragmentRange) { for (const DebugSubsectionRecord &R : FragmentRange) { if (Strings && Checksums) @@ -67,8 +72,8 @@ private: void initializeStrings(const DebugSubsectionRecord &SR); void initializeChecksums(const DebugSubsectionRecord &FCR); - std::unique_ptr<DebugStringTableSubsectionRef> OwnedStrings; - std::unique_ptr<DebugChecksumsSubsectionRef> OwnedChecksums; + std::shared_ptr<DebugStringTableSubsectionRef> OwnedStrings; + std::shared_ptr<DebugChecksumsSubsectionRef> OwnedChecksums; const DebugStringTableSubsectionRef *Strings = nullptr; const DebugChecksumsSubsectionRef *Checksums = nullptr; diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h index 5b6599d8c1db..b5479db97a15 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h @@ -46,6 +46,12 @@ public: return EC; return Error::success(); } + template <typename T> static Expected<T> deserializeAs(CVSymbol Symbol) { + T Record(Symbol.kind()); + if (auto EC = deserializeAs<T>(Symbol, Record)) + return std::move(EC); + return Record; + } explicit SymbolDeserializer(SymbolVisitorDelegate *Delegate, CodeViewContainer Container) diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h index f3086cf3dbb9..cf267f23967b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -363,12 +363,12 @@ public: : SymbolRecord(SymbolRecordKind::PublicSym32), RecordOffset(RecordOffset) {} - PublicSymFlags Flags; - uint32_t Offset; - uint16_t Segment; + PublicSymFlags Flags = PublicSymFlags::None; + uint32_t Offset = 0; + uint16_t Segment = 0; StringRef Name; - uint32_t RecordOffset; + uint32_t RecordOffset = 0; }; // S_REGISTER @@ -942,9 +942,14 @@ public: uint32_t RecordOffset; }; +// S_ANNOTATION + using CVSymbol = CVRecord<SymbolKind>; using CVSymbolArray = VarStreamArray<CVSymbol>; +Expected<CVSymbol> readSymbolFromStream(BinaryStreamRef Stream, + uint32_t Offset); + } // end namespace codeview } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h index b63ced5217b4..f4d8ab0c3c2e 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/SymbolSerializer.h @@ -28,7 +28,10 @@ namespace codeview { class SymbolSerializer : public SymbolVisitorCallbacks { BumpPtrAllocator &Storage; - std::vector<uint8_t> RecordBuffer; + // Since this is a fixed size buffer, use a stack allocated buffer. This + // yields measurable performance increase over the repeated heap allocations + // when serializing many independent records via writeOneSymbol. + std::array<uint8_t, MaxRecordLength> RecordBuffer; MutableBinaryByteStream Stream; BinaryStreamWriter Writer; SymbolRecordMapping Mapping; diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h index 0f856f57a727..e9fc9b0de8ef 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeCollection.h @@ -31,6 +31,16 @@ public: virtual bool contains(TypeIndex Index) = 0; virtual uint32_t size() = 0; virtual uint32_t capacity() = 0; + + template <typename TFunc> void ForEachRecord(TFunc Func) { + Optional<TypeIndex> Next = getFirst(); + + while (Next.hasValue()) { + TypeIndex N = *Next; + Func(N, getType(N)); + Next = getNext(N); + } + } }; } } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h index 965cdfd85f48..9887d901773a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h @@ -52,6 +52,19 @@ public: return Error::success(); } + template <typename T> + static Expected<T> deserializeAs(ArrayRef<uint8_t> Data) { + const RecordPrefix *Prefix = + reinterpret_cast<const RecordPrefix *>(Data.data()); + TypeRecordKind K = + static_cast<TypeRecordKind>(uint16_t(Prefix->RecordKind)); + T Record(K); + CVType CVT(static_cast<TypeLeafKind>(K), Data); + if (auto EC = deserializeAs<T>(CVT, Record)) + return std::move(EC); + return Record; + } + Error visitTypeBegin(CVType &Record) override { assert(!Mapping && "Already in a type mapping!"); Mapping = llvm::make_unique<MappingInfo>(Record.content()); diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h new file mode 100644 index 000000000000..741337533701 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h @@ -0,0 +1,204 @@ +//===- TypeHashing.h ---------------------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEHASHING_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPEHASHING_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/Hashing.h" + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeCollection.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" + +#include "llvm/Support/FormatProviders.h" + +#include <type_traits> + +namespace llvm { +namespace codeview { + +/// A locally hashed type represents a straightforward hash code of a serialized +/// record. The record is simply serialized, and then the bytes are hashed by +/// a standard algorithm. This is sufficient for the case of de-duplicating +/// records within a single sequence of types, because if two records both have +/// a back-reference to the same type in the same stream, they will both have +/// the same numeric value for the TypeIndex of the back reference. +struct LocallyHashedType { + hash_code Hash; + ArrayRef<uint8_t> RecordData; + + /// Given a type, compute its local hash. + static LocallyHashedType hashType(ArrayRef<uint8_t> RecordData); + + /// Given a sequence of types, compute all of the local hashes. + template <typename Range> + static std::vector<LocallyHashedType> hashTypes(Range &&Records) { + std::vector<LocallyHashedType> Hashes; + Hashes.reserve(std::distance(std::begin(Records), std::end(Records))); + for (const auto &R : Records) + Hashes.push_back(hashType(R)); + + return Hashes; + } + + static std::vector<LocallyHashedType> + hashTypeCollection(TypeCollection &Types) { + std::vector<LocallyHashedType> Hashes; + Types.ForEachRecord([&Hashes](TypeIndex TI, const CVType &Type) { + Hashes.push_back(hashType(Type.RecordData)); + }); + return Hashes; + } +}; + +enum class GlobalTypeHashAlg : uint16_t { SHA1 = 0 }; + +/// A globally hashed type represents a hash value that is sufficient to +/// uniquely identify a record across multiple type streams or type sequences. +/// This works by, for any given record A which references B, replacing the +/// TypeIndex that refers to B with a previously-computed global hash for B. As +/// this is a recursive algorithm (e.g. the global hash of B also depends on the +/// global hashes of the types that B refers to), a global hash can uniquely +/// identify identify that A occurs in another stream that has a completely +/// different graph structure. Although the hash itself is slower to compute, +/// probing is much faster with a globally hashed type, because the hash itself +/// is considered "as good as" the original type. Since type records can be +/// quite large, this makes the equality comparison of the hash much faster than +/// equality comparison of a full record. +struct GloballyHashedType { + GloballyHashedType() = default; + GloballyHashedType(StringRef H) + : GloballyHashedType(ArrayRef<uint8_t>(H.bytes_begin(), H.bytes_end())) {} + GloballyHashedType(ArrayRef<uint8_t> H) { + assert(H.size() == 20); + ::memcpy(Hash.data(), H.data(), 20); + } + std::array<uint8_t, 20> Hash; + + /// Given a sequence of bytes representing a record, compute a global hash for + /// this record. Due to the nature of global hashes incorporating the hashes + /// of referenced records, this function requires a list of types and ids + /// that RecordData might reference, indexable by TypeIndex. + static GloballyHashedType hashType(ArrayRef<uint8_t> RecordData, + ArrayRef<GloballyHashedType> PreviousTypes, + ArrayRef<GloballyHashedType> PreviousIds); + + /// Given a sequence of bytes representing a record, compute a global hash for + /// this record. Due to the nature of global hashes incorporating the hashes + /// of referenced records, this function requires a list of types and ids + /// that RecordData might reference, indexable by TypeIndex. + static GloballyHashedType hashType(CVType Type, + ArrayRef<GloballyHashedType> PreviousTypes, + ArrayRef<GloballyHashedType> PreviousIds) { + return hashType(Type.RecordData, PreviousTypes, PreviousIds); + } + + /// Given a sequence of combined type and ID records, compute global hashes + /// for each of them, returning the results in a vector of hashed types. + template <typename Range> + static std::vector<GloballyHashedType> hashTypes(Range &&Records) { + std::vector<GloballyHashedType> Hashes; + for (const auto &R : Records) + Hashes.push_back(hashType(R, Hashes, Hashes)); + + return Hashes; + } + + /// Given a sequence of combined type and ID records, compute global hashes + /// for each of them, returning the results in a vector of hashed types. + template <typename Range> + static std::vector<GloballyHashedType> + hashIds(Range &&Records, ArrayRef<GloballyHashedType> TypeHashes) { + std::vector<GloballyHashedType> IdHashes; + for (const auto &R : Records) + IdHashes.push_back(hashType(R, TypeHashes, IdHashes)); + + return IdHashes; + } + + static std::vector<GloballyHashedType> + hashTypeCollection(TypeCollection &Types) { + std::vector<GloballyHashedType> Hashes; + Types.ForEachRecord([&Hashes](TypeIndex TI, const CVType &Type) { + Hashes.push_back(hashType(Type.RecordData, Hashes, Hashes)); + }); + return Hashes; + } +}; +#if defined(_MSC_VER) +// is_trivially_copyable is not available in older versions of libc++, but it is +// available in all supported versions of MSVC, so at least this gives us some +// coverage. +static_assert(std::is_trivially_copyable<GloballyHashedType>::value, + "GloballyHashedType must be trivially copyable so that we can " + "reinterpret_cast arrays of hash data to arrays of " + "GloballyHashedType"); +#endif +} // namespace codeview + +template <> struct DenseMapInfo<codeview::LocallyHashedType> { + static codeview::LocallyHashedType Empty; + static codeview::LocallyHashedType Tombstone; + + static codeview::LocallyHashedType getEmptyKey() { return Empty; } + + static codeview::LocallyHashedType getTombstoneKey() { return Tombstone; } + + static unsigned getHashValue(codeview::LocallyHashedType Val) { + return Val.Hash; + } + + static bool isEqual(codeview::LocallyHashedType LHS, + codeview::LocallyHashedType RHS) { + if (LHS.Hash != RHS.Hash) + return false; + return LHS.RecordData == RHS.RecordData; + } +}; + +template <> struct DenseMapInfo<codeview::GloballyHashedType> { + static codeview::GloballyHashedType Empty; + static codeview::GloballyHashedType Tombstone; + + static codeview::GloballyHashedType getEmptyKey() { return Empty; } + + static codeview::GloballyHashedType getTombstoneKey() { return Tombstone; } + + static unsigned getHashValue(codeview::GloballyHashedType Val) { + return *reinterpret_cast<const unsigned *>(Val.Hash.data()); + } + + static bool isEqual(codeview::GloballyHashedType LHS, + codeview::GloballyHashedType RHS) { + return LHS.Hash == RHS.Hash; + } +}; + +template <> struct format_provider<codeview::LocallyHashedType> { +public: + static void format(const codeview::LocallyHashedType &V, + llvm::raw_ostream &Stream, StringRef Style) { + write_hex(Stream, V.Hash, HexPrintStyle::Upper, 8); + } +}; + +template <> struct format_provider<codeview::GloballyHashedType> { +public: + static void format(const codeview::GloballyHashedType &V, + llvm::raw_ostream &Stream, StringRef Style) { + for (uint8_t B : V.Hash) { + write_hex(Stream, B, HexPrintStyle::Upper, 2); + } + } +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h index e0c2226bdbd7..c71281de7145 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -98,6 +98,7 @@ public: static const uint32_t FirstNonSimpleIndex = 0x1000; static const uint32_t SimpleKindMask = 0x000000ff; static const uint32_t SimpleModeMask = 0x00000700; + static const uint32_t DecoratedItemIdMask = 0x80000000; public: TypeIndex() : Index(static_cast<uint32_t>(SimpleTypeKind::None)) {} @@ -110,6 +111,7 @@ public: uint32_t getIndex() const { return Index; } void setIndex(uint32_t I) { Index = I; } bool isSimple() const { return Index < FirstNonSimpleIndex; } + bool isDecoratedItemId() const { return !!(Index & DecoratedItemIdMask); } bool isNoneType() const { return *this == None(); } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h index afe8942159e8..c424a09ece89 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h @@ -30,11 +30,17 @@ void discoverTypeIndices(const CVType &Type, SmallVectorImpl<TiReference> &Refs); void discoverTypeIndices(const CVType &Type, SmallVectorImpl<TypeIndex> &Indices); +void discoverTypeIndices(ArrayRef<uint8_t> RecordData, + SmallVectorImpl<TypeIndex> &Indices); /// Discover type indices in symbol records. Returns false if this is an unknown /// record. -bool discoverTypeIndices(const CVSymbol &Symbol, - SmallVectorImpl<TiReference> &Refs); +bool discoverTypeIndicesInSymbol(const CVSymbol &Symbol, + SmallVectorImpl<TiReference> &Refs); +bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData, + SmallVectorImpl<TiReference> &Refs); +bool discoverTypeIndicesInSymbol(ArrayRef<uint8_t> RecordData, + SmallVectorImpl<TypeIndex> &Indices); } } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index 7942c0c0bc21..508bdd395f74 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -334,6 +334,11 @@ public: uint32_t Attrs; Optional<MemberPointerInfo> MemberInfo; + void setAttrs(PointerKind PK, PointerMode PM, PointerOptions PO, + uint8_t Size) { + Attrs = calcAttrs(PK, PM, PO, Size); + } + private: static uint32_t calcAttrs(PointerKind PK, PointerMode PM, PointerOptions PO, uint8_t Size) { @@ -412,6 +417,14 @@ public: return (Options & ClassOptions::HasUniqueName) != ClassOptions::None; } + bool isNested() const { + return (Options & ClassOptions::Nested) != ClassOptions::None; + } + + bool isForwardRef() const { + return (Options & ClassOptions::ForwardReference) != ClassOptions::None; + } + uint16_t getMemberCount() const { return MemberCount; } ClassOptions getOptions() const { return Options; } TypeIndex getFieldList() const { return FieldList; } diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h deleted file mode 100644 index 5a6507ee7f5b..000000000000 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h +++ /dev/null @@ -1,78 +0,0 @@ -//===- TypeRecordBuilder.h --------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPERECORDBUILDER_H -#define LLVM_DEBUGINFO_CODEVIEW_TYPERECORDBUILDER_H - -#include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" -#include "llvm/Support/EndianStream.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { -namespace codeview { - -class TypeRecordBuilder { -private: - TypeRecordBuilder(const TypeRecordBuilder &) = delete; - TypeRecordBuilder &operator=(const TypeRecordBuilder &) = delete; - -public: - explicit TypeRecordBuilder(TypeRecordKind Kind); - - void writeUInt8(uint8_t Value); - void writeInt16(int16_t Value); - void writeUInt16(uint16_t Value); - void writeInt32(int32_t Value); - void writeUInt32(uint32_t Value); - void writeInt64(int64_t Value); - void writeUInt64(uint64_t Value); - void writeTypeIndex(TypeIndex TypeInd); - void writeTypeRecordKind(TypeRecordKind Kind); - void writeEncodedInteger(int64_t Value); - void writeEncodedSignedInteger(int64_t Value); - void writeEncodedUnsignedInteger(uint64_t Value); - void writeNullTerminatedString(StringRef Value); - void writeGuid(StringRef Guid); - void writeBytes(StringRef Value) { Stream << Value; } - - llvm::StringRef str(); - - uint64_t size() const { return Stream.tell(); } - TypeRecordKind kind() const { return Kind; } - - /// Returns the number of bytes remaining before this record is larger than - /// the maximum record length. Accounts for the extra two byte size field in - /// the header. - size_t maxBytesRemaining() const { return MaxRecordLength - size() - 2; } - - void truncate(uint64_t Size) { - // This works because raw_svector_ostream is not buffered. - assert(Size < Buffer.size()); - Buffer.resize(Size); - } - - void reset(TypeRecordKind K) { - Buffer.clear(); - Kind = K; - writeTypeRecordKind(K); - } - -private: - TypeRecordKind Kind; - llvm::SmallVector<char, 256> Buffer; - llvm::raw_svector_ostream Stream; - llvm::support::endian::Writer<llvm::support::endianness::little> Writer; -}; -} -} - -#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeSerializer.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeSerializer.h deleted file mode 100644 index 0e734a8170bd..000000000000 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeSerializer.h +++ /dev/null @@ -1,159 +0,0 @@ -//===- TypeSerializer.h -----------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZER_H -#define LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/RecordSerialization.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" -#include "llvm/DebugInfo/CodeView/TypeRecordMapping.h" -#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/BinaryStreamWriter.h" -#include "llvm/Support/Error.h" -#include <cassert> -#include <cstdint> -#include <memory> -#include <vector> - -namespace llvm { -namespace codeview { - -class TypeHasher; - -class TypeSerializer : public TypeVisitorCallbacks { - struct SubRecord { - SubRecord(TypeLeafKind K, uint32_t S) : Kind(K), Size(S) {} - - TypeLeafKind Kind; - uint32_t Size = 0; - }; - struct RecordSegment { - SmallVector<SubRecord, 16> SubRecords; - - uint32_t length() const { - uint32_t L = sizeof(RecordPrefix); - for (const auto &R : SubRecords) { - L += R.Size; - } - return L; - } - }; - - using MutableRecordList = SmallVector<MutableArrayRef<uint8_t>, 2>; - - static constexpr uint8_t ContinuationLength = 8; - BumpPtrAllocator &RecordStorage; - RecordSegment CurrentSegment; - MutableRecordList FieldListSegments; - - Optional<TypeLeafKind> TypeKind; - Optional<TypeLeafKind> MemberKind; - std::vector<uint8_t> RecordBuffer; - MutableBinaryByteStream Stream; - BinaryStreamWriter Writer; - TypeRecordMapping Mapping; - - /// Private type record hashing implementation details are handled here. - std::unique_ptr<TypeHasher> Hasher; - - /// Contains a list of all records indexed by TypeIndex.toArrayIndex(). - SmallVector<ArrayRef<uint8_t>, 2> SeenRecords; - - /// Temporary storage that we use to copy a record's data while re-writing - /// its type indices. - SmallVector<uint8_t, 256> RemapStorage; - - TypeIndex nextTypeIndex() const; - - bool isInFieldList() const; - MutableArrayRef<uint8_t> getCurrentSubRecordData(); - MutableArrayRef<uint8_t> getCurrentRecordData(); - Error writeRecordPrefix(TypeLeafKind Kind); - - Expected<MutableArrayRef<uint8_t>> - addPadding(MutableArrayRef<uint8_t> Record); - -public: - explicit TypeSerializer(BumpPtrAllocator &Storage, bool Hash = true); - ~TypeSerializer() override; - - void reset(); - - BumpPtrAllocator &getAllocator() { return RecordStorage; } - - ArrayRef<ArrayRef<uint8_t>> records() const; - TypeIndex insertRecordBytes(ArrayRef<uint8_t> &Record); - TypeIndex insertRecord(const RemappedType &Record); - Expected<TypeIndex> visitTypeEndGetIndex(CVType &Record); - - using TypeVisitorCallbacks::visitTypeBegin; - Error visitTypeBegin(CVType &Record) override; - Error visitTypeEnd(CVType &Record) override; - Error visitMemberBegin(CVMemberRecord &Record) override; - Error visitMemberEnd(CVMemberRecord &Record) override; - -#define TYPE_RECORD(EnumName, EnumVal, Name) \ - virtual Error visitKnownRecord(CVType &CVR, Name##Record &Record) override { \ - return visitKnownRecordImpl(CVR, Record); \ - } -#define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName) -#define MEMBER_RECORD(EnumName, EnumVal, Name) \ - Error visitKnownMember(CVMemberRecord &CVR, Name##Record &Record) override { \ - return visitKnownMemberImpl<Name##Record>(CVR, Record); \ - } -#define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName) -#include "llvm/DebugInfo/CodeView/CodeViewTypes.def" - -private: - template <typename RecordKind> - Error visitKnownRecordImpl(CVType &CVR, RecordKind &Record) { - return Mapping.visitKnownRecord(CVR, Record); - } - - template <typename RecordType> - Error visitKnownMemberImpl(CVMemberRecord &CVR, RecordType &Record) { - assert(CVR.Kind == static_cast<TypeLeafKind>(Record.getKind())); - - if (auto EC = Writer.writeEnum(CVR.Kind)) - return EC; - - if (auto EC = Mapping.visitKnownMember(CVR, Record)) - return EC; - - // Get all the data that was just written and is yet to be committed to - // the current segment. Then pad it to 4 bytes. - MutableArrayRef<uint8_t> ThisRecord = getCurrentSubRecordData(); - auto ExpectedRecord = addPadding(ThisRecord); - if (!ExpectedRecord) - return ExpectedRecord.takeError(); - ThisRecord = *ExpectedRecord; - - CurrentSegment.SubRecords.emplace_back(CVR.Kind, ThisRecord.size()); - CVR.Data = ThisRecord; - - // Both the last subrecord and the total length of this segment should be - // multiples of 4. - assert(ThisRecord.size() % 4 == 0); - assert(CurrentSegment.length() % 4 == 0); - - return Error::success(); - } -}; - -} // end namespace codeview -} // end namespace llvm - -#endif // LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h index d78fab47db66..59e216abcb11 100644 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h +++ b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h @@ -19,7 +19,9 @@ namespace llvm { namespace codeview { class TypeIndex; -class TypeTableBuilder; +struct GloballyHashedType; +class GlobalTypeTableBuilder; +class MergingTypeTableBuilder; /// \brief Merge one set of type records into another. This method assumes /// that all records are type records, and there are no Id records present. @@ -34,7 +36,7 @@ class TypeTableBuilder; /// /// \returns Error::success() if the operation succeeded, otherwise an /// appropriate error code. -Error mergeTypeRecords(TypeTableBuilder &Dest, +Error mergeTypeRecords(MergingTypeTableBuilder &Dest, SmallVectorImpl<TypeIndex> &SourceToDest, const CVTypeArray &Types); @@ -59,7 +61,7 @@ Error mergeTypeRecords(TypeTableBuilder &Dest, /// /// \returns Error::success() if the operation succeeded, otherwise an /// appropriate error code. -Error mergeIdRecords(TypeTableBuilder &Dest, ArrayRef<TypeIndex> Types, +Error mergeIdRecords(MergingTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types, SmallVectorImpl<TypeIndex> &SourceToDest, const CVTypeArray &Ids); @@ -78,11 +80,27 @@ Error mergeIdRecords(TypeTableBuilder &Dest, ArrayRef<TypeIndex> Types, /// /// \returns Error::success() if the operation succeeded, otherwise an /// appropriate error code. -Error mergeTypeAndIdRecords(TypeTableBuilder &DestIds, - TypeTableBuilder &DestTypes, +Error mergeTypeAndIdRecords(MergingTypeTableBuilder &DestIds, + MergingTypeTableBuilder &DestTypes, SmallVectorImpl<TypeIndex> &SourceToDest, const CVTypeArray &IdsAndTypes); +Error mergeTypeAndIdRecords(GlobalTypeTableBuilder &DestIds, + GlobalTypeTableBuilder &DestTypes, + SmallVectorImpl<TypeIndex> &SourceToDest, + const CVTypeArray &IdsAndTypes, + ArrayRef<GloballyHashedType> Hashes); + +Error mergeTypeRecords(GlobalTypeTableBuilder &Dest, + SmallVectorImpl<TypeIndex> &SourceToDest, + const CVTypeArray &Types, + ArrayRef<GloballyHashedType> Hashes); + +Error mergeIdRecords(GlobalTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types, + SmallVectorImpl<TypeIndex> &SourceToDest, + const CVTypeArray &Ids, + ArrayRef<GloballyHashedType> Hashes); + } // end namespace codeview } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h b/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h deleted file mode 100644 index 1069dcd45334..000000000000 --- a/contrib/llvm/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h +++ /dev/null @@ -1,137 +0,0 @@ -//===- TypeTableBuilder.h ---------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H -#define LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/DebugInfo/CodeView/CodeView.h" -#include "llvm/DebugInfo/CodeView/TypeIndex.h" -#include "llvm/DebugInfo/CodeView/TypeRecord.h" -#include "llvm/DebugInfo/CodeView/TypeSerializer.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/Error.h" -#include <algorithm> -#include <cassert> -#include <cstdint> -#include <type_traits> - -namespace llvm { -namespace codeview { - -class TypeTableBuilder { -private: - TypeIndex handleError(Error EC) const { - assert(false && "Couldn't write Type!"); - consumeError(std::move(EC)); - return TypeIndex(); - } - - BumpPtrAllocator &Allocator; - TypeSerializer Serializer; - -public: - explicit TypeTableBuilder(BumpPtrAllocator &Allocator, - bool WriteUnique = true) - : Allocator(Allocator), Serializer(Allocator, WriteUnique) {} - TypeTableBuilder(const TypeTableBuilder &) = delete; - TypeTableBuilder &operator=(const TypeTableBuilder &) = delete; - - bool empty() const { return Serializer.records().empty(); } - - BumpPtrAllocator &getAllocator() const { return Allocator; } - - template <typename T> TypeIndex writeKnownType(T &Record) { - static_assert(!std::is_same<T, FieldListRecord>::value, - "Can't serialize FieldList!"); - - CVType Type; - Type.Type = static_cast<TypeLeafKind>(Record.getKind()); - if (auto EC = Serializer.visitTypeBegin(Type)) - return handleError(std::move(EC)); - if (auto EC = Serializer.visitKnownRecord(Type, Record)) - return handleError(std::move(EC)); - - auto ExpectedIndex = Serializer.visitTypeEndGetIndex(Type); - if (!ExpectedIndex) - return handleError(ExpectedIndex.takeError()); - - return *ExpectedIndex; - } - - TypeIndex writeSerializedRecord(ArrayRef<uint8_t> Record) { - return Serializer.insertRecordBytes(Record); - } - - TypeIndex writeSerializedRecord(const RemappedType &Record) { - return Serializer.insertRecord(Record); - } - - template <typename TFunc> void ForEachRecord(TFunc Func) { - uint32_t Index = TypeIndex::FirstNonSimpleIndex; - - for (auto Record : Serializer.records()) { - Func(TypeIndex(Index), Record); - ++Index; - } - } - - ArrayRef<ArrayRef<uint8_t>> records() const { return Serializer.records(); } -}; - -class FieldListRecordBuilder { - TypeTableBuilder &TypeTable; - BumpPtrAllocator Allocator; - TypeSerializer TempSerializer; - CVType Type; - -public: - explicit FieldListRecordBuilder(TypeTableBuilder &TypeTable) - : TypeTable(TypeTable), TempSerializer(Allocator, false) { - Type.Type = TypeLeafKind::LF_FIELDLIST; - } - - void begin() { - TempSerializer.reset(); - - if (auto EC = TempSerializer.visitTypeBegin(Type)) - consumeError(std::move(EC)); - } - - template <typename T> void writeMemberType(T &Record) { - CVMemberRecord CVMR; - CVMR.Kind = static_cast<TypeLeafKind>(Record.getKind()); - if (auto EC = TempSerializer.visitMemberBegin(CVMR)) - consumeError(std::move(EC)); - if (auto EC = TempSerializer.visitKnownMember(CVMR, Record)) - consumeError(std::move(EC)); - if (auto EC = TempSerializer.visitMemberEnd(CVMR)) - consumeError(std::move(EC)); - } - - TypeIndex end(bool Write) { - TypeIndex Index; - if (auto EC = TempSerializer.visitTypeEnd(Type)) { - consumeError(std::move(EC)); - return TypeIndex(); - } - - if (Write) { - for (auto Record : TempSerializer.records()) - Index = TypeTable.writeSerializedRecord(Record); - } - - return Index; - } -}; - -} // end namespace codeview -} // end namespace llvm - -#endif // LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H diff --git a/contrib/llvm/include/llvm/DebugInfo/DIContext.h b/contrib/llvm/include/llvm/DebugInfo/DIContext.h index 936813dc6abc..abace9378607 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DIContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DIContext.h @@ -17,6 +17,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> #include <memory> @@ -26,9 +27,7 @@ namespace llvm { -class raw_ostream; - -/// DILineInfo - a format-neutral container for source line information. +/// A format-neutral container for source line information. struct DILineInfo { std::string FileName; std::string FunctionName; @@ -46,20 +45,35 @@ struct DILineInfo { FileName == RHS.FileName && FunctionName == RHS.FunctionName && StartLine == RHS.StartLine && Discriminator == RHS.Discriminator; } + bool operator!=(const DILineInfo &RHS) const { return !(*this == RHS); } + bool operator<(const DILineInfo &RHS) const { return std::tie(FileName, FunctionName, Line, Column, StartLine, Discriminator) < std::tie(RHS.FileName, RHS.FunctionName, RHS.Line, RHS.Column, RHS.StartLine, RHS.Discriminator); } + + explicit operator bool() const { return *this != DILineInfo(); } + + void dump(raw_ostream &OS) { + OS << "Line info: "; + if (FileName != "<invalid>") + OS << "file '" << FileName << "', "; + if (FunctionName != "<invalid>") + OS << "function '" << FunctionName << "', "; + OS << "line " << Line << ", "; + OS << "column " << Column << ", "; + OS << "start line " << StartLine << '\n'; + } }; using DILineInfoTable = SmallVector<std::pair<uint64_t, DILineInfo>, 16>; -/// DIInliningInfo - a format-neutral container for inlined code description. +/// A format-neutral container for inlined code description. class DIInliningInfo { SmallVector<DILineInfo, 4> Frames; @@ -85,7 +99,7 @@ public: } }; -/// DIGlobal - container for description of a global variable. +/// Container for description of a global variable. struct DIGlobal { std::string Name; uint64_t Start = 0; @@ -98,8 +112,8 @@ struct DIGlobal { /// preference regarding the type of name resolution the caller wants. enum class DINameKind { None, ShortName, LinkageName }; -/// DILineInfoSpecifier - controls which fields of DILineInfo container -/// should be filled with data. +/// Controls which fields of DILineInfo container should be filled +/// with data. struct DILineInfoSpecifier { enum class FileLineInfoKind { None, Default, AbsoluteFilePath }; using FunctionNameKind = DINameKind; @@ -112,48 +126,54 @@ struct DILineInfoSpecifier { : FLIKind(FLIKind), FNKind(FNKind) {} }; +/// This is just a helper to programmatically construct DIDumpType. +enum DIDumpTypeCounter { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME) \ + DIDT_ID_##ENUM_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION + DIDT_ID_UUID, + DIDT_ID_Count +}; +static_assert(DIDT_ID_Count <= 32, "section types overflow storage"); + /// Selects which debug sections get dumped. -enum DIDumpType { +enum DIDumpType : unsigned { DIDT_Null, - DIDT_All, - DIDT_Abbrev, - DIDT_AbbrevDwo, - DIDT_Aranges, - DIDT_Frames, - DIDT_Info, - DIDT_InfoDwo, - DIDT_Types, - DIDT_TypesDwo, - DIDT_Line, - DIDT_LineDwo, - DIDT_Loc, - DIDT_LocDwo, - DIDT_Macro, - DIDT_Ranges, - DIDT_Pubnames, - DIDT_Pubtypes, - DIDT_GnuPubnames, - DIDT_GnuPubtypes, - DIDT_Str, - DIDT_StrOffsets, - DIDT_StrDwo, - DIDT_StrOffsetsDwo, - DIDT_AppleNames, - DIDT_AppleTypes, - DIDT_AppleNamespaces, - DIDT_AppleObjC, - DIDT_CUIndex, - DIDT_GdbIndex, - DIDT_TUIndex, + DIDT_All = ~0U, +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME) \ + DIDT_##ENUM_NAME = 1U << DIDT_ID_##ENUM_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION + DIDT_UUID = 1 << DIDT_ID_UUID, }; /// Container for dump options that control which debug information will be /// dumped. struct DIDumpOptions { - DIDumpType DumpType = DIDT_All; - bool DumpEH = false; - bool SummarizeTypes = false; - bool Brief = false; + unsigned DumpType = DIDT_All; + unsigned RecurseDepth = -1U; + bool ShowAddresses = true; + bool ShowChildren = false; + bool ShowParents = false; + bool ShowForm = false; + bool SummarizeTypes = false; + bool Verbose = false; + + /// Return default option set for printing a single DIE without children. + static DIDumpOptions getForSingleDIE() { + DIDumpOptions Opts; + Opts.RecurseDepth = 0; + return Opts; + } + + /// Return the options with RecurseDepth set to 0 unless explicitly required. + DIDumpOptions noImplicitRecursion() const { + DIDumpOptions Opts = *this; + if (RecurseDepth == -1U && !ShowChildren) + Opts.RecurseDepth = 0; + return Opts; + } }; class DIContext { @@ -170,7 +190,7 @@ public: virtual void dump(raw_ostream &OS, DIDumpOptions DumpOpts) = 0; - virtual bool verify(raw_ostream &OS, DIDumpType DumpType = DIDT_All) { + virtual bool verify(raw_ostream &OS, DIDumpOptions DumpOpts = {}) { // No verifier? Just say things went well. return true; } @@ -202,22 +222,23 @@ public: /// Calculate the address of the given section. /// The section need not be present in the local address space. The addresses /// need to be consistent with the addresses used to query the DIContext and - /// the output of this function should be deterministic, i.e. repeated calls with - /// the same Sec should give the same address. + /// the output of this function should be deterministic, i.e. repeated calls + /// with the same Sec should give the same address. virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const { return 0; } /// If conveniently available, return the content of the given Section. /// - /// When the section is available in the local address space, in relocated (loaded) - /// form, e.g. because it was relocated by a JIT for execution, this function - /// should provide the contents of said section in `Data`. If the loaded section - /// is not available, or the cost of retrieving it would be prohibitive, this - /// function should return false. In that case, relocations will be read from the - /// local (unrelocated) object file and applied on the fly. Note that this method - /// is used purely for optimzation purposes in the common case of JITting in the - /// local address space, so returning false should always be correct. + /// When the section is available in the local address space, in relocated + /// (loaded) form, e.g. because it was relocated by a JIT for execution, this + /// function should provide the contents of said section in `Data`. If the + /// loaded section is not available, or the cost of retrieving it would be + /// prohibitive, this function should return false. In that case, relocations + /// will be read from the local (unrelocated) object file and applied on the + /// fly. Note that this method is used purely for optimzation purposes in the + /// common case of JITting in the local address space, so returning false + /// should always be correct. virtual bool getLoadedSectionContents(const object::SectionRef &Sec, StringRef &Data) const { return false; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h index 190a69b75739..84b23398b8cc 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h @@ -28,31 +28,54 @@ class raw_ostream; class DWARFAbbreviationDeclaration { public: struct AttributeSpec { - AttributeSpec(dwarf::Attribute A, dwarf::Form F, Optional<int64_t> V) - : Attr(A), Form(F), ByteSizeOrValue(V) {} + AttributeSpec(dwarf::Attribute A, dwarf::Form F, int64_t Value) + : Attr(A), Form(F), Value(Value) { + assert(isImplicitConst()); + } + AttributeSpec(dwarf::Attribute A, dwarf::Form F, Optional<uint8_t> ByteSize) + : Attr(A), Form(F) { + assert(!isImplicitConst()); + this->ByteSize.HasByteSize = ByteSize.hasValue(); + if (this->ByteSize.HasByteSize) + this->ByteSize.ByteSize = *ByteSize; + } dwarf::Attribute Attr; dwarf::Form Form; + private: /// The following field is used for ByteSize for non-implicit_const /// attributes and as value for implicit_const ones, indicated by /// Form == DW_FORM_implicit_const. /// The following cases are distinguished: - /// * Form != DW_FORM_implicit_const and ByteSizeOrValue has a value: - /// ByteSizeOrValue contains the fixed size in bytes - /// for the Form in this object. - /// * Form != DW_FORM_implicit_const and ByteSizeOrValue is None: + /// * Form != DW_FORM_implicit_const and HasByteSize is true: + /// ByteSize contains the fixed size in bytes for the Form in this + /// object. + /// * Form != DW_FORM_implicit_const and HasByteSize is false: /// byte size of Form either varies according to the DWARFUnit /// that it is contained in or the value size varies and must be /// decoded from the debug information in order to determine its size. /// * Form == DW_FORM_implicit_const: - /// ByteSizeOrValue contains value for the implicit_const attribute. - Optional<int64_t> ByteSizeOrValue; - + /// Value contains value for the implicit_const attribute. + struct ByteSizeStorage { + bool HasByteSize; + uint8_t ByteSize; + }; + union { + ByteSizeStorage ByteSize; + int64_t Value; + }; + + public: bool isImplicitConst() const { return Form == dwarf::DW_FORM_implicit_const; } + int64_t getImplicitConstValue() const { + assert(isImplicitConst()); + return Value; + } + /// Get the fixed byte size of this Form if possible. This function might /// use the DWARFUnit to calculate the size of the Form, like for /// DW_AT_address and DW_AT_ref_addr, so this isn't just an accessor for diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h index eb6d0f541c1e..0bade10f6201 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h @@ -13,7 +13,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" -#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" +#include "llvm/DebugInfo/DWARF/DWARFFormValue.h" #include <cstdint> #include <utility> @@ -21,6 +21,9 @@ namespace llvm { class raw_ostream; +/// This implements the Apple accelerator table format, a precursor of the +/// DWARF 5 accelerator table format. +/// TODO: Factor out a common base class for both formats. class DWARFAcceleratorTable { struct Header { uint32_t Magic; @@ -43,13 +46,51 @@ class DWARFAcceleratorTable { struct HeaderData HdrData; DWARFDataExtractor AccelSection; DataExtractor StringSection; + bool IsValid = false; public: + /// An iterator for the entries associated with one key. Each entry can have + /// multiple DWARFFormValues. + class ValueIterator : public std::iterator<std::input_iterator_tag, + ArrayRef<DWARFFormValue>> { + const DWARFAcceleratorTable *AccelTable = nullptr; + SmallVector<DWARFFormValue, 3> AtomForms; ///< The decoded data entry. + + unsigned DataOffset = 0; ///< Offset into the section. + unsigned Data = 0; ///< Current data entry. + unsigned NumData = 0; ///< Number of data entries. + + /// Advance the iterator. + void Next(); + public: + /// Construct a new iterator for the entries at \p DataOffset. + ValueIterator(const DWARFAcceleratorTable &AccelTable, unsigned DataOffset); + /// End marker. + ValueIterator() = default; + + const ArrayRef<DWARFFormValue> operator*() const { + return AtomForms; + } + ValueIterator &operator++() { Next(); return *this; } + ValueIterator operator++(int) { + ValueIterator I = *this; + Next(); + return I; + } + friend bool operator==(const ValueIterator &A, const ValueIterator &B) { + return A.NumData == B.NumData && A.DataOffset == B.DataOffset; + } + friend bool operator!=(const ValueIterator &A, const ValueIterator &B) { + return !(A == B); + } + }; + + DWARFAcceleratorTable(const DWARFDataExtractor &AccelSection, DataExtractor StringSection) : AccelSection(AccelSection), StringSection(StringSection) {} - bool extract(); + llvm::Error extract(); uint32_t getNumBuckets(); uint32_t getNumHashes(); uint32_t getSizeHdr(); @@ -61,12 +102,15 @@ public: /// performing a lookup by name. /// /// \param HashDataOffset an offset into the hash data table - /// \returns DIEOffset the offset into the .debug_info section for the DIE - /// related to the input hash data offset. Currently this function returns - /// only the DIEOffset but it can be modified to return more data regarding - /// the DIE - uint32_t readAtoms(uint32_t &HashDataOffset); + /// \returns <DieOffset, DieTag> + /// DieOffset is the offset into the .debug_info section for the DIE + /// related to the input hash data offset. + /// DieTag is the tag of the DIE + std::pair<uint32_t, dwarf::Tag> readAtoms(uint32_t &HashDataOffset); void dump(raw_ostream &OS) const; + + /// Look up all entries in the accelerator table matching \c Key. + iterator_range<ValueIterator> equal_range(StringRef Key) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index ee2e805050c0..2ddbc4b91ba2 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -17,6 +17,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h" #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h" #include "llvm/DebugInfo/DWARF/DWARFDebugAranges.h" @@ -26,6 +27,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" #include "llvm/DebugInfo/DWARF/DWARFGdbIndex.h" +#include "llvm/DebugInfo/DWARF/DWARFObject.h" #include "llvm/DebugInfo/DWARF/DWARFSection.h" #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" @@ -42,13 +44,18 @@ namespace llvm { class DataExtractor; +class MCRegisterInfo; class MemoryBuffer; class raw_ostream; +/// Used as a return value for a error callback passed to DWARF context. +/// Callback should return Halt if client application wants to stop +/// object parsing, or should return Continue otherwise. +enum class ErrorPolicy { Halt, Continue }; + /// DWARFContext /// This data structure is the top level entity that deals with dwarf debug -/// information parsing. The actual data is supplied through pure virtual -/// methods that a concrete implementation provides. +/// information parsing. The actual data is supplied through DWARFObj. class DWARFContext : public DIContext { DWARFUnitSection<DWARFCompileUnit> CUs; std::deque<DWARFUnitSection<DWARFTypeUnit>> TUs; @@ -62,6 +69,10 @@ class DWARFContext : public DIContext { std::unique_ptr<DWARFDebugFrame> DebugFrame; std::unique_ptr<DWARFDebugFrame> EHFrame; std::unique_ptr<DWARFDebugMacro> Macro; + std::unique_ptr<DWARFAcceleratorTable> AppleNames; + std::unique_ptr<DWARFAcceleratorTable> AppleTypes; + std::unique_ptr<DWARFAcceleratorTable> AppleNamespaces; + std::unique_ptr<DWARFAcceleratorTable> AppleObjC; DWARFUnitSection<DWARFCompileUnit> DWOCUs; std::deque<DWARFUnitSection<DWARFTypeUnit>> DWOTUs; @@ -78,6 +89,9 @@ class DWARFContext : public DIContext { StringMap<std::weak_ptr<DWOFile>> DWOFiles; std::weak_ptr<DWOFile> DWP; bool CheckedForDWP = false; + std::string DWPName; + + std::unique_ptr<MCRegisterInfo> RegInfo; /// Read compile units from the debug_info section (if necessary) /// and store them in CUs. @@ -95,18 +109,34 @@ class DWARFContext : public DIContext { /// and store them in DWOTUs. void parseDWOTypeUnits(); +protected: + std::unique_ptr<const DWARFObject> DObj; + public: - DWARFContext() : DIContext(CK_DWARF) {} + DWARFContext(std::unique_ptr<const DWARFObject> DObj, + std::string DWPName = ""); + ~DWARFContext(); + DWARFContext(DWARFContext &) = delete; DWARFContext &operator=(DWARFContext &) = delete; + const DWARFObject &getDWARFObj() const { return *DObj; } + static bool classof(const DIContext *DICtx) { return DICtx->getKind() == CK_DWARF; } - void dump(raw_ostream &OS, DIDumpOptions DumpOpts) override; + /// Dump a textual representation to \p OS. If any \p DumpOffsets are present, + /// dump only the record at the specified offset. + void dump(raw_ostream &OS, DIDumpOptions DumpOpts, + std::array<Optional<uint64_t>, DIDT_ID_Count> DumpOffsets); - bool verify(raw_ostream &OS, DIDumpType DumpType = DIDT_All) override; + void dump(raw_ostream &OS, DIDumpOptions DumpOpts) override { + std::array<Optional<uint64_t>, DIDT_ID_Count> DumpOffsets; + dump(OS, DumpOpts, DumpOffsets); + } + + bool verify(raw_ostream &OS, DIDumpOptions DumpOpts = {}) override; using cu_iterator_range = DWARFUnitSection<DWARFCompileUnit>::iterator_range; using tu_iterator_range = DWARFUnitSection<DWARFTypeUnit>::iterator_range; @@ -212,9 +242,33 @@ public: /// Get a pointer to the parsed DebugMacro object. const DWARFDebugMacro *getDebugMacro(); + /// Get a reference to the parsed accelerator table object. + const DWARFAcceleratorTable &getAppleNames(); + + /// Get a reference to the parsed accelerator table object. + const DWARFAcceleratorTable &getAppleTypes(); + + /// Get a reference to the parsed accelerator table object. + const DWARFAcceleratorTable &getAppleNamespaces(); + + /// Get a reference to the parsed accelerator table object. + const DWARFAcceleratorTable &getAppleObjC(); + /// Get a pointer to a parsed line table corresponding to a compile unit. const DWARFDebugLine::LineTable *getLineTableForUnit(DWARFUnit *cu); + /// Wraps the returned DIEs for a given address. + struct DIEsForAddress { + DWARFCompileUnit *CompileUnit = nullptr; + DWARFDie FunctionDIE; + DWARFDie BlockDIE; + explicit operator bool() const { return CompileUnit != nullptr; } + }; + + /// Get the compilation unit, the function DIE and lexical block DIE for the + /// given address where applicable. + DIEsForAddress getDIEsForAddress(uint64_t Address); + DILineInfo getLineInfoForAddress(uint64_t Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; DILineInfoTable getLineInfoForAddressRange(uint64_t Address, uint64_t Size, @@ -222,198 +276,39 @@ public: DIInliningInfo getInliningInfoForAddress(uint64_t Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; - virtual StringRef getFileName() const = 0; - virtual bool isLittleEndian() const = 0; - virtual uint8_t getAddressSize() const = 0; - virtual const DWARFSection &getInfoSection() = 0; - virtual void forEachTypesSections(function_ref<void(DWARFSection &)> F) = 0; - virtual StringRef getAbbrevSection() = 0; - virtual const DWARFSection &getLocSection() = 0; - virtual StringRef getARangeSection() = 0; - virtual StringRef getDebugFrameSection() = 0; - virtual StringRef getEHFrameSection() = 0; - virtual const DWARFSection &getLineSection() = 0; - virtual StringRef getStringSection() = 0; - virtual const DWARFSection& getRangeSection() = 0; - virtual StringRef getMacinfoSection() = 0; - virtual StringRef getPubNamesSection() = 0; - virtual StringRef getPubTypesSection() = 0; - virtual StringRef getGnuPubNamesSection() = 0; - virtual StringRef getGnuPubTypesSection() = 0; - - /// DWARF v5 - /// @{ - virtual const DWARFSection &getStringOffsetSection() = 0; - /// @} - - // Sections for DWARF5 split dwarf proposal. - virtual const DWARFSection &getInfoDWOSection() = 0; - virtual void - forEachTypesDWOSections(function_ref<void(DWARFSection &)> F) = 0; - virtual StringRef getAbbrevDWOSection() = 0; - virtual const DWARFSection &getLineDWOSection() = 0; - virtual const DWARFSection &getLocDWOSection() = 0; - virtual StringRef getStringDWOSection() = 0; - virtual const DWARFSection &getStringOffsetDWOSection() = 0; - virtual const DWARFSection &getRangeDWOSection() = 0; - virtual const DWARFSection &getAddrSection() = 0; - virtual const DWARFSection& getAppleNamesSection() = 0; - virtual const DWARFSection& getAppleTypesSection() = 0; - virtual const DWARFSection& getAppleNamespacesSection() = 0; - virtual const DWARFSection& getAppleObjCSection() = 0; - virtual StringRef getCUIndexSection() = 0; - virtual StringRef getGdbIndexSection() = 0; - virtual StringRef getTUIndexSection() = 0; - + bool isLittleEndian() const { return DObj->isLittleEndian(); } static bool isSupportedVersion(unsigned version) { return version == 2 || version == 3 || version == 4 || version == 5; } std::shared_ptr<DWARFContext> getDWOContext(StringRef AbsolutePath); -private: - /// Return the compile unit that includes an offset (relative to .debug_info). - DWARFCompileUnit *getCompileUnitForOffset(uint32_t Offset); - - /// Return the compile unit which contains instruction with provided - /// address. - DWARFCompileUnit *getCompileUnitForAddress(uint64_t Address); -}; - -/// Used as a return value for a error callback passed to DWARF context. -/// Callback should return Halt if client application wants to stop -/// object parsing, or should return Continue otherwise. -enum class ErrorPolicy { Halt, Continue }; - -/// DWARFContextInMemory is the simplest possible implementation of a -/// DWARFContext. It assumes all content is available in memory and stores -/// pointers to it. -class DWARFContextInMemory : public DWARFContext { - virtual void anchor(); - - using TypeSectionMap = MapVector<object::SectionRef, DWARFSection, - std::map<object::SectionRef, unsigned>>; - - StringRef FileName; - bool IsLittleEndian; - uint8_t AddressSize; - DWARFSection InfoSection; - TypeSectionMap TypesSections; - StringRef AbbrevSection; - DWARFSection LocSection; - StringRef ARangeSection; - StringRef DebugFrameSection; - StringRef EHFrameSection; - DWARFSection LineSection; - StringRef StringSection; - DWARFSection RangeSection; - StringRef MacinfoSection; - StringRef PubNamesSection; - StringRef PubTypesSection; - StringRef GnuPubNamesSection; - StringRef GnuPubTypesSection; - - /// DWARF v5 - /// @{ - DWARFSection StringOffsetSection; - /// @} - - // Sections for DWARF5 split dwarf proposal. - DWARFSection InfoDWOSection; - TypeSectionMap TypesDWOSections; - StringRef AbbrevDWOSection; - DWARFSection LineDWOSection; - DWARFSection LocDWOSection; - StringRef StringDWOSection; - DWARFSection StringOffsetDWOSection; - DWARFSection RangeDWOSection; - DWARFSection AddrSection; - DWARFSection AppleNamesSection; - DWARFSection AppleTypesSection; - DWARFSection AppleNamespacesSection; - DWARFSection AppleObjCSection; - StringRef CUIndexSection; - StringRef GdbIndexSection; - StringRef TUIndexSection; - - SmallVector<SmallString<32>, 4> UncompressedSections; - - DWARFSection *mapNameToDWARFSection(StringRef Name); - StringRef *mapSectionToMember(StringRef Name); - - /// If Sec is compressed section, decompresses and updates its contents - /// provided by Data. Otherwise leaves it unchanged. - Error maybeDecompress(const object::SectionRef &Sec, StringRef Name, - StringRef &Data); + const MCRegisterInfo *getRegisterInfo() const { return RegInfo.get(); } /// Function used to handle default error reporting policy. Prints a error /// message and returns Continue, so DWARF context ignores the error. static ErrorPolicy defaultErrorHandler(Error E); + static std::unique_ptr<DWARFContext> + create(const object::ObjectFile &Obj, const LoadedObjectInfo *L = nullptr, + function_ref<ErrorPolicy(Error)> HandleError = defaultErrorHandler, + std::string DWPName = ""); -public: - DWARFContextInMemory( - const object::ObjectFile &Obj, const LoadedObjectInfo *L = nullptr, - function_ref<ErrorPolicy(Error)> HandleError = defaultErrorHandler); - - DWARFContextInMemory(const StringMap<std::unique_ptr<MemoryBuffer>> &Sections, - uint8_t AddrSize, - bool isLittleEndian = sys::IsLittleEndianHost); - - StringRef getFileName() const override { return FileName; } - bool isLittleEndian() const override { return IsLittleEndian; } - uint8_t getAddressSize() const override { return AddressSize; } - const DWARFSection &getInfoSection() override { return InfoSection; } - void forEachTypesSections(function_ref<void(DWARFSection &)> F) override { - for (auto &P : TypesSections) - F(P.second); - } - StringRef getAbbrevSection() override { return AbbrevSection; } - const DWARFSection &getLocSection() override { return LocSection; } - StringRef getARangeSection() override { return ARangeSection; } - StringRef getDebugFrameSection() override { return DebugFrameSection; } - StringRef getEHFrameSection() override { return EHFrameSection; } - const DWARFSection &getLineSection() override { return LineSection; } - StringRef getStringSection() override { return StringSection; } - const DWARFSection &getRangeSection() override { return RangeSection; } - StringRef getMacinfoSection() override { return MacinfoSection; } - StringRef getPubNamesSection() override { return PubNamesSection; } - StringRef getPubTypesSection() override { return PubTypesSection; } - StringRef getGnuPubNamesSection() override { return GnuPubNamesSection; } - StringRef getGnuPubTypesSection() override { return GnuPubTypesSection; } - const DWARFSection& getAppleNamesSection() override { return AppleNamesSection; } - const DWARFSection& getAppleTypesSection() override { return AppleTypesSection; } - const DWARFSection& getAppleNamespacesSection() override { return AppleNamespacesSection; } - const DWARFSection& getAppleObjCSection() override { return AppleObjCSection; } - - // DWARF v5 - const DWARFSection &getStringOffsetSection() override { - return StringOffsetSection; - } - - // Sections for DWARF5 split dwarf proposal. - const DWARFSection &getInfoDWOSection() override { return InfoDWOSection; } - - void forEachTypesDWOSections(function_ref<void(DWARFSection &)> F) override { - for (auto &P : TypesDWOSections) - F(P.second); - } - - StringRef getAbbrevDWOSection() override { return AbbrevDWOSection; } - const DWARFSection &getLineDWOSection() override { return LineDWOSection; } - const DWARFSection &getLocDWOSection() override { return LocDWOSection; } - StringRef getStringDWOSection() override { return StringDWOSection; } - - const DWARFSection &getStringOffsetDWOSection() override { - return StringOffsetDWOSection; - } + static std::unique_ptr<DWARFContext> + create(const StringMap<std::unique_ptr<MemoryBuffer>> &Sections, + uint8_t AddrSize, bool isLittleEndian = sys::IsLittleEndianHost); - const DWARFSection &getRangeDWOSection() override { return RangeDWOSection; } + /// Loads register info for the architecture of the provided object file. + /// Improves readability of dumped DWARF expressions. Requires the caller to + /// have initialized the relevant target descriptions. + Error loadRegisterInfo(const object::ObjectFile &Obj); - const DWARFSection &getAddrSection() override { return AddrSection; } +private: + /// Return the compile unit that includes an offset (relative to .debug_info). + DWARFCompileUnit *getCompileUnitForOffset(uint32_t Offset); - StringRef getCUIndexSection() override { return CUIndexSection; } - StringRef getGdbIndexSection() override { return GdbIndexSection; } - StringRef getTUIndexSection() override { return TUIndexSection; } + /// Return the compile unit which contains instruction with provided + /// address. + DWARFCompileUnit *getCompileUnitForAddress(uint64_t Address); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h index ef4360f66621..a379d9c85b38 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h @@ -14,18 +14,21 @@ #include "llvm/Support/DataExtractor.h" namespace llvm { +class DWARFObject; /// A DataExtractor (typically for an in-memory copy of an object-file section) /// plus a relocation map for that section, if there is one. class DWARFDataExtractor : public DataExtractor { - const RelocAddrMap *RelocMap = nullptr; + const DWARFObject *Obj = nullptr; + const DWARFSection *Section = nullptr; + public: /// Constructor for the normal case of extracting data from a DWARF section. /// The DWARFSection's lifetime must be at least as long as the extractor's. - DWARFDataExtractor(const DWARFSection &Section, bool IsLittleEndian, - uint8_t AddressSize) - : DataExtractor(Section.Data, IsLittleEndian, AddressSize), - RelocMap(&Section.Relocs) {} + DWARFDataExtractor(const DWARFObject &Obj, const DWARFSection &Section, + bool IsLittleEndian, uint8_t AddressSize) + : DataExtractor(Section.Data, IsLittleEndian, AddressSize), Obj(&Obj), + Section(&Section) {} /// Constructor for cases when there are no relocations. DWARFDataExtractor(StringRef Data, bool IsLittleEndian, uint8_t AddressSize) diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h index 65571598d743..d277ec382ba5 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h @@ -56,8 +56,9 @@ class DWARFDebugAbbrev { using DWARFAbbreviationDeclarationSetMap = std::map<uint64_t, DWARFAbbreviationDeclarationSet>; - DWARFAbbreviationDeclarationSetMap AbbrDeclSets; + mutable DWARFAbbreviationDeclarationSetMap AbbrDeclSets; mutable DWARFAbbreviationDeclarationSetMap::const_iterator PrevAbbrOffsetPos; + mutable Optional<DataExtractor> Data; public: DWARFDebugAbbrev(); @@ -66,9 +67,11 @@ public: getAbbreviationDeclarationSet(uint64_t CUAbbrOffset) const; void dump(raw_ostream &OS) const; + void parse() const; void extract(DataExtractor Data); DWARFAbbreviationDeclarationSetMap::const_iterator begin() const { + parse(); return AbbrDeclSets.begin(); } diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index e0a779bb8182..a711fb295444 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -29,13 +29,19 @@ public: DWARFDebugFrame(bool IsEH); ~DWARFDebugFrame(); - /// \brief Dump the section data into the given stream. - void dump(raw_ostream &OS) const; + /// Dump the section data into the given stream. + void dump(raw_ostream &OS, Optional<uint64_t> Offset) const; /// \brief Parse the section from raw data. /// data is assumed to be pointing to the beginning of the section. void parse(DataExtractor Data); + /// Return whether the section has any entries. + bool empty() const { return Entries.empty(); } + + /// Return the entry at the given offset or nullptr. + FrameEntry *getEntryAtOffset(uint64_t Offset) const; + private: std::vector<std::unique_ptr<FrameEntry>> Entries; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index 0c8f98aa62f9..de8ad4e5ef3c 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -15,6 +15,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" +#include "llvm/Support/MD5.h" #include <cstdint> #include <map> #include <string> @@ -22,6 +23,7 @@ namespace llvm { +class DWARFUnit; class raw_ostream; class DWARFDebugLine { @@ -33,6 +35,7 @@ public: uint64_t DirIdx = 0; uint64_t ModTime = 0; uint64_t Length = 0; + MD5::MD5Result Checksum; }; struct Prologue { @@ -45,11 +48,11 @@ public: /// parameters affect interpretation of forms (used in the directory and /// file tables starting with v5). DWARFFormParams FormParams; - /// In v5, size in bytes of a segment selector. - uint8_t SegSelectorSize; /// The number of bytes following the prologue_length field to the beginning /// of the first byte of the statement program itself. uint64_t PrologueLength; + /// In v5, size in bytes of a segment selector. + uint8_t SegSelectorSize; /// The size in bytes of the smallest target machine instruction. Statement /// program opcodes that alter the address register first multiply their /// operands by this value. @@ -65,6 +68,8 @@ public: uint8_t LineRange; /// The number assigned to the first special opcode. uint8_t OpcodeBase; + /// For v5, whether filename entries provide an MD5 checksum. + bool HasMD5; std::vector<uint8_t> StandardOpcodeLengths; std::vector<StringRef> IncludeDirectories; std::vector<FileNameEntry> FileNames; @@ -95,7 +100,8 @@ public: void clear(); void dump(raw_ostream &OS) const; - bool parse(const DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr); + bool parse(const DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, + const DWARFUnit *U = nullptr); }; /// Standard .debug_line state machine structure. @@ -217,7 +223,8 @@ public: void clear(); /// Parse prologue and all rows. - bool parse(const DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr); + bool parse(DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, + const DWARFUnit *U, raw_ostream *OS = nullptr); using RowVector = std::vector<Row>; using RowIter = RowVector::const_iterator; @@ -234,8 +241,8 @@ public: }; const LineTable *getLineTable(uint32_t Offset) const; - const LineTable *getOrParseLineTable(const DWARFDataExtractor &DebugLineData, - uint32_t Offset); + const LineTable *getOrParseLineTable(DWARFDataExtractor &DebugLineData, + uint32_t Offset, const DWARFUnit *U); private: struct ParsingState { diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index c2b8d0cd73d8..a6d319a90457 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -10,16 +10,19 @@ #ifndef LLVM_DEBUGINFO_DWARF_DWARFDEBUGLOC_H #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGLOC_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include <cstdint> namespace llvm { - +class DWARFUnit; +class MCRegisterInfo; class raw_ostream; class DWARFDebugLoc { +public: /// A single location within a location list. struct Entry { /// The beginning address of the instruction range. @@ -27,7 +30,7 @@ class DWARFDebugLoc { /// The ending address of the instruction range. uint64_t End; /// The location of the variable within the specified range. - SmallVector<unsigned char, 4> Loc; + SmallVector<char, 4> Loc; }; /// A list of locations that contain one variable. @@ -37,42 +40,72 @@ class DWARFDebugLoc { unsigned Offset; /// All the locations in which the variable is stored. SmallVector<Entry, 2> Entries; + /// Dump this list on OS. + void dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, + const MCRegisterInfo *MRI, unsigned Indent) const; }; +private: using LocationLists = SmallVector<LocationList, 4>; /// A list of all the variables in the debug_loc section, each one describing /// the locations in which the variable is stored. LocationLists Locations; + unsigned AddressSize; + + bool IsLittleEndian; + public: /// Print the location lists found within the debug_loc section. - void dump(raw_ostream &OS) const; + void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, + Optional<uint64_t> Offset) const; /// Parse the debug_loc section accessible via the 'data' parameter using the /// address size also given in 'data' to interpret the address ranges. void parse(const DWARFDataExtractor &data); + + /// Return the location list at the given offset or nullptr. + LocationList const *getLocationListAtOffset(uint64_t Offset) const; + + Optional<LocationList> parseOneLocationList(DWARFDataExtractor Data, + uint32_t *Offset); }; class DWARFDebugLocDWO { +public: struct Entry { uint64_t Start; uint32_t Length; - SmallVector<unsigned char, 4> Loc; + SmallVector<char, 4> Loc; }; struct LocationList { unsigned Offset; SmallVector<Entry, 2> Entries; + void dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, + const MCRegisterInfo *RegInfo, unsigned Indent) const; }; +private: using LocationLists = SmallVector<LocationList, 4>; LocationLists Locations; + unsigned AddressSize; + + bool IsLittleEndian; + public: void parse(DataExtractor data); - void dump(raw_ostream &OS) const; + void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, + Optional<uint64_t> Offset) const; + + /// Return the location list at the given offset or nullptr. + LocationList const *getLocationListAtOffset(uint64_t Offset) const; + + static Optional<LocationList> parseOneLocationList(DataExtractor Data, + uint32_t *Offset); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h index 135c50761e36..bfe2fc3ac02d 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h @@ -53,6 +53,9 @@ public: /// Parse the debug_macinfo section accessible via the 'data' parameter. void parse(DataExtractor data); + + /// Return whether the section has any entries. + bool empty() const { return Macros.empty(); } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h index a309fd104f93..761871dc6255 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h @@ -69,7 +69,7 @@ private: public: DWARFDebugPubTable(StringRef Data, bool LittleEndian, bool GnuStyle); - void dump(StringRef Name, raw_ostream &OS) const; + void dump(raw_ostream &OS) const; ArrayRef<Set> getData() { return Sets; } }; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h index bcba14b1630d..f9ec96366a53 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h @@ -18,14 +18,47 @@ namespace llvm { +struct BaseAddress; class raw_ostream; struct DWARFAddressRange { uint64_t LowPC; uint64_t HighPC; uint64_t SectionIndex; + + DWARFAddressRange() = default; + + /// Used for unit testing. + DWARFAddressRange(uint64_t LowPC, uint64_t HighPC, uint64_t SectionIndex = 0) + : LowPC(LowPC), HighPC(HighPC), SectionIndex(SectionIndex) {} + + /// Returns true if LowPC is smaller or equal to HighPC. This accounts for + /// dead-stripped ranges. + bool valid() const { return LowPC <= HighPC; } + + /// Returns true if [LowPC, HighPC) intersects with [RHS.LowPC, RHS.HighPC). + bool intersects(const DWARFAddressRange &RHS) const { + // Empty ranges can't intersect. + if (LowPC == HighPC || RHS.LowPC == RHS.HighPC) + return false; + return (LowPC < RHS.HighPC) && (HighPC > RHS.LowPC); + } + + /// Returns true if [LowPC, HighPC) fully contains [RHS.LowPC, RHS.HighPC). + bool contains(const DWARFAddressRange &RHS) const { + if (LowPC <= RHS.LowPC && RHS.LowPC <= HighPC) + return LowPC <= RHS.HighPC && RHS.HighPC <= HighPC; + return false; + } }; +static inline bool operator<(const DWARFAddressRange &LHS, + const DWARFAddressRange &RHS) { + return std::tie(LHS.LowPC, LHS.HighPC) < std::tie(RHS.LowPC, RHS.HighPC); +} + +raw_ostream &operator<<(raw_ostream &OS, const DWARFAddressRange &R); + /// DWARFAddressRangesVector - represents a set of absolute address ranges. using DWARFAddressRangesVector = std::vector<DWARFAddressRange>; @@ -85,7 +118,8 @@ public: /// getAbsoluteRanges - Returns absolute address ranges defined by this range /// list. Has to be passed base address of the compile unit referencing this /// range list. - DWARFAddressRangesVector getAbsoluteRanges(uint64_t BaseAddress) const; + DWARFAddressRangesVector + getAbsoluteRanges(llvm::Optional<BaseAddress> BaseAddr) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h index b216491b615a..75fc5995c5b2 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -108,21 +108,19 @@ public: /// /// \returns a valid DWARFDie instance if this object has children or an /// invalid DWARFDie instance if it doesn't. - DWARFDie getFirstChild() const { - if (isValid() && Die->hasChildren()) - return DWARFDie(U, Die + 1); - return DWARFDie(); - } + DWARFDie getFirstChild() const; /// Dump the DIE and all of its attributes to the supplied stream. /// /// \param OS the stream to use for output. - /// \param recurseDepth the depth to recurse to when dumping this DIE and its - /// children. /// \param indent the number of characters to indent each line that is output. - void dump(raw_ostream &OS, unsigned recurseDepth, unsigned indent = 0, + void dump(raw_ostream &OS, unsigned indent = 0, DIDumpOptions DumpOpts = DIDumpOptions()) const; + + /// Convenience zero-argument overload for debugging. + LLVM_DUMP_METHOD void dump() const; + /// Extract the specified attribute from this DIE. /// /// Extract an attribute value from this DIE only. This call doesn't look @@ -304,6 +302,10 @@ inline bool operator!=(const DWARFDie &LHS, const DWARFDie &RHS) { return !(LHS == RHS); } +inline bool operator<(const DWARFDie &LHS, const DWARFDie &RHS) { + return LHS.getOffset() < RHS.getOffset(); +} + class DWARFDie::iterator : public iterator_facade_base<iterator, std::forward_iterator_tag, const DWARFDie> { diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h new file mode 100644 index 000000000000..dcd486f3fb13 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h @@ -0,0 +1,151 @@ +//===--- DWARFExpression.h - DWARF Expression handling ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_DWARFEXPRESSION_H +#define LLVM_DEBUGINFO_DWARFEXPRESSION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/DataExtractor.h" + +namespace llvm { +class DWARFUnit; +class MCRegisterInfo; +class raw_ostream; + +class DWARFExpression { +public: + class iterator; + + /// This class represents an Operation in the Expression. Each operation can + /// have up to 2 oprerands. + /// + /// An Operation can be in Error state (check with isError()). This + /// means that it couldn't be decoded successfully and if it is the + /// case, all others fields contain undefined values. + class Operation { + public: + /// Size and signedness of expression operations' operands. + enum Encoding : uint8_t { + Size1 = 0, + Size2 = 1, + Size4 = 2, + Size8 = 3, + SizeLEB = 4, + SizeAddr = 5, + SizeRefAddr = 6, + SizeBlock = 7, ///< Preceding operand contains block size + SignBit = 0x8, + SignedSize1 = SignBit | Size1, + SignedSize2 = SignBit | Size2, + SignedSize4 = SignBit | Size4, + SignedSize8 = SignBit | Size8, + SignedSizeLEB = SignBit | SizeLEB, + SizeNA = 0xFF ///< Unused operands get this encoding. + }; + + enum DwarfVersion : uint8_t { + DwarfNA, ///< Serves as a marker for unused entries + Dwarf2 = 2, + Dwarf3, + Dwarf4 + }; + + /// Description of the encoding of one expression Op. + struct Description { + DwarfVersion Version; ///< Dwarf version where the Op was introduced. + Encoding Op[2]; ///< Encoding for Op operands, or SizeNA. + + Description(DwarfVersion Version = DwarfNA, Encoding Op1 = SizeNA, + Encoding Op2 = SizeNA) + : Version(Version) { + Op[0] = Op1; + Op[1] = Op2; + } + }; + + private: + friend class DWARFExpression::iterator; + uint8_t Opcode; ///< The Op Opcode, DW_OP_<something>. + Description Desc; + bool Error; + uint32_t EndOffset; + uint64_t Operands[2]; + + public: + Description &getDescription() { return Desc; } + uint8_t getCode() { return Opcode; } + uint64_t getRawOperand(unsigned Idx) { return Operands[Idx]; } + uint32_t getEndOffset() { return EndOffset; } + bool extract(DataExtractor Data, uint16_t Version, uint8_t AddressSize, + uint32_t Offset); + bool isError() { return Error; } + bool print(raw_ostream &OS, const DWARFExpression *U, + const MCRegisterInfo *RegInfo, bool isEH); + }; + + /// An iterator to go through the expression operations. + class iterator + : public iterator_facade_base<iterator, std::forward_iterator_tag, Operation> { + friend class DWARFExpression; + DWARFExpression *Expr; + uint32_t Offset; + Operation Op; + iterator(DWARFExpression *Expr, uint32_t Offset) + : Expr(Expr), Offset(Offset) { + Op.Error = + Offset >= Expr->Data.getData().size() || + !Op.extract(Expr->Data, Expr->Version, Expr->AddressSize, Offset); + } + + public: + class Operation &operator++() { + Offset = Op.isError() ? Expr->Data.getData().size() : Op.EndOffset; + Op.Error = + Offset >= Expr->Data.getData().size() || + !Op.extract(Expr->Data, Expr->Version, Expr->AddressSize, Offset); + return Op; + } + + class Operation &operator*() { + return Op; + } + + // Comparison operators are provided out of line. + friend bool operator==(const iterator &, const iterator &); + }; + + DWARFExpression(DataExtractor Data, uint16_t Version, uint8_t AddressSize) + : Data(Data), Version(Version), AddressSize(AddressSize) { + assert(AddressSize == 8 || AddressSize == 4); + } + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, Data.getData().size()); } + + void print(raw_ostream &OS, const MCRegisterInfo *RegInfo); + +private: + DataExtractor Data; + uint16_t Version; + uint8_t AddressSize; +}; + +inline bool operator==(const DWARFExpression::iterator &LHS, + const DWARFExpression::iterator &RHS) { + return LHS.Expr == RHS.Expr && LHS.Offset == RHS.Offset; +} + +inline bool operator!=(const DWARFExpression::iterator &LHS, + const DWARFExpression::iterator &RHS) { + return !(LHS == RHS); +} +} +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 008dba9b42ac..d32053519ec4 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -14,6 +14,7 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include <cstdint> @@ -101,18 +102,14 @@ public: bool isFormClass(FormClass FC) const; const DWARFUnit *getUnit() const { return U; } - void dump(raw_ostream &OS) const; + void dump(raw_ostream &OS, DIDumpOptions DumpOpts = DIDumpOptions()) const; - /// Extracts a value in \p Data at offset \p *OffsetPtr. - /// - /// The passed DWARFUnit is allowed to be nullptr, in which case some - /// kind of forms that depend on Unit information are disallowed. - /// \param Data The DWARFDataExtractor to use. - /// \param OffsetPtr The offset within \p Data where the data starts. - /// \param U The optional DWARFUnit supplying information for some forms. - /// \returns whether the extraction succeeded. + /// Extracts a value in \p Data at offset \p *OffsetPtr. The information + /// in \p FormParams is needed to interpret some forms. The optional + /// \p Unit allows extracting information if the form refers to other + /// sections (e.g., .debug_str). bool extractValue(const DWARFDataExtractor &Data, uint32_t *OffsetPtr, - const DWARFUnit *U); + DWARFFormParams FormParams, const DWARFUnit *U = nullptr); bool isInlinedCStr() const { return Value.data != nullptr && Value.data == (const uint8_t *)Value.cstr; diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h new file mode 100644 index 000000000000..167eb2da5ba0 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h @@ -0,0 +1,79 @@ +//===- DWARFObject.h --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===-----------------------------------------------------------------------===/ + +#ifndef LLVM_DEBUGINFO_DWARF_DWARFOBJECT_H +#define LLVM_DEBUGINFO_DWARF_DWARFOBJECT_H + +#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" +#include "llvm/DebugInfo/DWARF/DWARFSection.h" +#include "llvm/Object/ObjectFile.h" + +namespace llvm { +// This is responsible for low level access to the object file. It +// knows how to find the required sections and compute relocated +// values. +// The default implementations of the get<Section> methods return dummy values. +// This is to allow clients that only need some of those to implement just the +// ones they need. We can't use unreachable for as many cases because the parser +// implementation is eager and will call some of these methods even if the +// result is not used. +class DWARFObject { + DWARFSection Dummy; + +public: + virtual ~DWARFObject() = default; + virtual StringRef getFileName() const { llvm_unreachable("unimplemented"); } + virtual const object::ObjectFile *getFile() const { return nullptr; } + virtual ArrayRef<SectionName> getSectionNames() const { return {}; } + virtual bool isLittleEndian() const = 0; + virtual uint8_t getAddressSize() const { llvm_unreachable("unimplemented"); } + virtual const DWARFSection &getInfoSection() const { return Dummy; } + virtual void + forEachTypesSections(function_ref<void(const DWARFSection &)> F) const {} + virtual StringRef getAbbrevSection() const { return ""; } + virtual const DWARFSection &getLocSection() const { return Dummy; } + virtual StringRef getARangeSection() const { return ""; } + virtual StringRef getDebugFrameSection() const { return ""; } + virtual StringRef getEHFrameSection() const { return ""; } + virtual const DWARFSection &getLineSection() const { return Dummy; } + virtual StringRef getStringSection() const { return ""; } + virtual const DWARFSection &getRangeSection() const { return Dummy; } + virtual StringRef getMacinfoSection() const { return ""; } + virtual StringRef getPubNamesSection() const { return ""; } + virtual StringRef getPubTypesSection() const { return ""; } + virtual StringRef getGnuPubNamesSection() const { return ""; } + virtual StringRef getGnuPubTypesSection() const { return ""; } + virtual const DWARFSection &getStringOffsetSection() const { return Dummy; } + virtual const DWARFSection &getInfoDWOSection() const { return Dummy; } + virtual void + forEachTypesDWOSections(function_ref<void(const DWARFSection &)> F) const {} + virtual StringRef getAbbrevDWOSection() const { return ""; } + virtual const DWARFSection &getLineDWOSection() const { return Dummy; } + virtual const DWARFSection &getLocDWOSection() const { return Dummy; } + virtual StringRef getStringDWOSection() const { return ""; } + virtual const DWARFSection &getStringOffsetDWOSection() const { + return Dummy; + } + virtual const DWARFSection &getRangeDWOSection() const { return Dummy; } + virtual const DWARFSection &getAddrSection() const { return Dummy; } + virtual const DWARFSection &getAppleNamesSection() const { return Dummy; } + virtual const DWARFSection &getAppleTypesSection() const { return Dummy; } + virtual const DWARFSection &getAppleNamespacesSection() const { + return Dummy; + } + virtual const DWARFSection &getAppleObjCSection() const { return Dummy; } + virtual StringRef getCUIndexSection() const { return ""; } + virtual StringRef getGdbIndexSection() const { return ""; } + virtual StringRef getTUIndexSection() const { return ""; } + virtual Optional<RelocAddrEntry> find(const DWARFSection &Sec, + uint64_t Pos) const = 0; +}; + +} // namespace llvm +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h index 2b8a53a4c93e..77045f0794ae 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFSection.h @@ -11,13 +11,16 @@ #define LLVM_DEBUGINFO_DWARF_DWARFSECTION_H #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" namespace llvm { struct DWARFSection { StringRef Data; - RelocAddrMap Relocs; +}; + +struct SectionName { + StringRef Name; + bool IsNameUnique; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index 4a5793ecb8fa..a7842454f435 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -42,7 +42,7 @@ public: return DWARFUnit::getHeaderSize() + 12; } - void dump(raw_ostream &OS, bool Brief = false); + void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}); static const DWARFSectionKind Section = DW_SECT_TYPES; protected: diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h index 056c1b77c65d..e9178e03fa8a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -47,10 +47,11 @@ public: /// Returns the Unit that contains the given section offset in the /// same section this Unit originated from. virtual DWARFUnit *getUnitForOffset(uint32_t Offset) const = 0; + virtual DWARFUnit *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E) = 0; void parse(DWARFContext &C, const DWARFSection &Section); void parseDWO(DWARFContext &C, const DWARFSection &DWOSection, - DWARFUnitIndex *Index = nullptr); + bool Lazy = false); protected: ~DWARFUnitSectionBase() = default; @@ -59,7 +60,7 @@ protected: const DWARFDebugAbbrev *DA, const DWARFSection *RS, StringRef SS, const DWARFSection &SOS, const DWARFSection *AOS, const DWARFSection &LS, - bool isLittleEndian, bool isDWO) = 0; + bool isLittleEndian, bool isDWO, bool Lazy) = 0; }; const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context, @@ -70,6 +71,7 @@ template<typename UnitType> class DWARFUnitSection final : public SmallVector<std::unique_ptr<UnitType>, 1>, public DWARFUnitSectionBase { bool Parsed = false; + std::function<std::unique_ptr<UnitType>(uint32_t)> Parser; public: using UnitVector = SmallVectorImpl<std::unique_ptr<UnitType>>; @@ -82,34 +84,87 @@ public: [](uint32_t LHS, const std::unique_ptr<UnitType> &RHS) { return LHS < RHS->getNextUnitOffset(); }); - if (CU != this->end()) + if (CU != this->end() && (*CU)->getOffset() <= Offset) return CU->get(); return nullptr; } + UnitType *getUnitForIndexEntry(const DWARFUnitIndex::Entry &E) override { + const auto *CUOff = E.getOffset(DW_SECT_INFO); + if (!CUOff) + return nullptr; + + auto Offset = CUOff->Offset; + + auto *CU = std::upper_bound( + this->begin(), this->end(), CUOff->Offset, + [](uint32_t LHS, const std::unique_ptr<UnitType> &RHS) { + return LHS < RHS->getNextUnitOffset(); + }); + if (CU != this->end() && (*CU)->getOffset() <= Offset) + return CU->get(); + + if (!Parser) + return nullptr; + + auto U = Parser(Offset); + if (!U) + U = nullptr; + + auto *NewCU = U.get(); + this->insert(CU, std::move(U)); + return NewCU; + } private: void parseImpl(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, const DWARFSection *RS, StringRef SS, const DWARFSection &SOS, const DWARFSection *AOS, - const DWARFSection &LS, bool LE, bool IsDWO) override { + const DWARFSection &LS, bool LE, bool IsDWO, + bool Lazy) override { if (Parsed) return; - const auto &Index = getDWARFUnitIndex(Context, UnitType::Section); DataExtractor Data(Section.Data, LE, 0); + if (!Parser) { + const DWARFUnitIndex *Index = nullptr; + if (IsDWO) + Index = &getDWARFUnitIndex(Context, UnitType::Section); + Parser = [=, &Context, &Section, &SOS, + &LS](uint32_t Offset) -> std::unique_ptr<UnitType> { + if (!Data.isValidOffset(Offset)) + return nullptr; + auto U = llvm::make_unique<UnitType>( + Context, Section, DA, RS, SS, SOS, AOS, LS, LE, IsDWO, *this, + Index ? Index->getFromOffset(Offset) : nullptr); + if (!U->extract(Data, &Offset)) + return nullptr; + return U; + }; + } + if (Lazy) + return; + auto I = this->begin(); uint32_t Offset = 0; while (Data.isValidOffset(Offset)) { - auto U = llvm::make_unique<UnitType>(Context, Section, DA, RS, SS, SOS, - AOS, LS, LE, IsDWO, *this, - Index.getFromOffset(Offset)); - if (!U->extract(Data, &Offset)) + if (I != this->end() && (*I)->getOffset() == Offset) { + ++I; + continue; + } + auto U = Parser(Offset); + if (!U) break; - this->push_back(std::move(U)); - Offset = this->back()->getNextUnitOffset(); + Offset = U->getNextUnitOffset(); + I = std::next(this->insert(I, std::move(U))); } Parsed = true; } }; +/// Represents base address of the CU. +struct BaseAddress { + uint64_t Address; + uint64_t SectionIndex; +}; + class DWARFUnit { DWARFContext &Context; /// Section containing this DWARFUnit. @@ -123,7 +178,7 @@ class DWARFUnit { const DWARFSection &StringOffsetSection; uint64_t StringOffsetSectionBase = 0; const DWARFSection *AddrOffsetSection; - uint32_t AddrOffsetSectionBase; + uint32_t AddrOffsetSectionBase = 0; bool isLittleEndian; bool isDWO; const DWARFUnitSectionBase &UnitSection; @@ -133,9 +188,10 @@ class DWARFUnit { uint32_t Offset; uint32_t Length; - const DWARFAbbreviationDeclarationSet *Abbrevs; + mutable const DWARFAbbreviationDeclarationSet *Abbrevs; + uint64_t AbbrOffset; uint8_t UnitType; - uint64_t BaseAddr; + llvm::Optional<BaseAddress> BaseAddr; /// The compile unit debug information entry items. std::vector<DWARFDebugInfoEntry> DieArray; @@ -197,19 +253,12 @@ public: bool getAddrOffsetSectionItem(uint32_t Index, uint64_t &Result) const; bool getStringOffsetSectionItem(uint32_t Index, uint64_t &Result) const; - DWARFDataExtractor getDebugInfoExtractor() const { - return DWARFDataExtractor(InfoSection, isLittleEndian, - getAddressByteSize()); - } + DWARFDataExtractor getDebugInfoExtractor() const; DataExtractor getStringExtractor() const { return DataExtractor(StringSection, false, 0); } - const RelocAddrMap *getRelocMap() const { return &InfoSection.Relocs; } - const RelocAddrMap &getStringOffsetsRelocMap() const { - return StringOffsetSection.Relocs; - } bool extract(DataExtractor debug_info, uint32_t* offset_ptr); @@ -232,18 +281,25 @@ public: return FormParams.getDwarfOffsetByteSize(); } - const DWARFAbbreviationDeclarationSet *getAbbreviations() const { - return Abbrevs; - } + const DWARFAbbreviationDeclarationSet *getAbbreviations() const; uint8_t getUnitType() const { return UnitType; } - static bool isValidUnitType(uint8_t UnitType) { - return UnitType == dwarf::DW_UT_compile || UnitType == dwarf::DW_UT_type || - UnitType == dwarf::DW_UT_partial || - UnitType == dwarf::DW_UT_skeleton || - UnitType == dwarf::DW_UT_split_compile || - UnitType == dwarf::DW_UT_split_type; + static bool isMatchingUnitTypeAndTag(uint8_t UnitType, dwarf::Tag Tag) { + switch (UnitType) { + case dwarf::DW_UT_compile: + return Tag == dwarf::DW_TAG_compile_unit; + case dwarf::DW_UT_type: + return Tag == dwarf::DW_TAG_type_unit; + case dwarf::DW_UT_partial: + return Tag == dwarf::DW_TAG_partial_unit; + case dwarf::DW_UT_skeleton: + return Tag == dwarf::DW_TAG_skeleton_unit; + case dwarf::DW_UT_split_compile: + case dwarf::DW_UT_split_type: + return dwarf::isUnitType(Tag); + } + return false; } /// \brief Return the number of bytes for the header of a unit of @@ -266,11 +322,9 @@ public: llvm_unreachable("Invalid UnitType."); } - uint64_t getBaseAddress() const { return BaseAddr; } + llvm::Optional<BaseAddress> getBaseAddress() const { return BaseAddr; } - void setBaseAddress(uint64_t base_addr) { - BaseAddr = base_addr; - } + void setBaseAddress(BaseAddress BaseAddr) { this->BaseAddr = BaseAddr; } DWARFDie getUnitDIE(bool ExtractUnitDIEOnly = true) { extractDIEsIfNeeded(ExtractUnitDIEOnly); @@ -284,6 +338,11 @@ public: void collectAddressRanges(DWARFAddressRangesVector &CURanges); + /// Returns subprogram DIE with address range encompassing the provided + /// address. The pointer is alive as long as parsed compile unit DIEs are not + /// cleared. + DWARFDie getSubroutineForAddress(uint64_t Address); + /// getInlinedChainForAddress - fetches inlined chain for a given address. /// Returns empty chain if there is no subprogram containing address. The /// chain is valid as long as parsed compile unit DIEs are not cleared. @@ -318,6 +377,7 @@ public: DWARFDie getParent(const DWARFDebugInfoEntry *Die); DWARFDie getSibling(const DWARFDebugInfoEntry *Die); + DWARFDie getFirstChild(const DWARFDebugInfoEntry *Die); /// \brief Return the DIE object for a given offset inside the /// unit's DIE vector. @@ -366,11 +426,6 @@ private: /// parseDWO - Parses .dwo file for current compile unit. Returns true if /// it was actually constructed. bool parseDWO(); - - /// getSubroutineForAddress - Returns subprogram DIE with address range - /// encompassing the provided address. The pointer is alive as long as parsed - /// compile unit DIEs are not cleared. - DWARFDie getSubroutineForAddress(uint64_t Address); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h index 8e2ce023695b..49ed4bb222f3 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -83,9 +83,13 @@ public: DWARFUnitIndex(DWARFSectionKind InfoColumnKind) : InfoColumnKind(InfoColumnKind) {} + explicit operator bool() const { return Header.NumBuckets; } + bool parse(DataExtractor IndexData); void dump(raw_ostream &OS) const; + const Entry *getFromOffset(uint32_t Offset) const; + const Entry *getFromHash(uint64_t Offset) const; ArrayRef<DWARFSectionKind> getColumnKinds() const { return makeArrayRef(ColumnKinds.get(), Header.NumColumns); diff --git a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h index c0291a83ed97..0d920abe3231 100644 --- a/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h +++ b/contrib/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h @@ -10,6 +10,10 @@ #ifndef LLVM_DEBUGINFO_DWARF_DWARFVERIFIER_H #define LLVM_DEBUGINFO_DWARF_DWARFVERIFIER_H +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" +#include "llvm/DebugInfo/DWARF/DWARFDie.h" + #include <cstdint> #include <map> #include <set> @@ -22,17 +26,91 @@ class DWARFDie; class DWARFUnit; class DWARFAcceleratorTable; class DWARFDataExtractor; +class DWARFDebugAbbrev; +class DataExtractor; +struct DWARFSection; /// A class that verifies DWARF debug information given a DWARF Context. class DWARFVerifier { +public: + /// A class that keeps the address range information for a single DIE. + struct DieRangeInfo { + DWARFDie Die; + + /// Sorted DWARFAddressRanges. + std::vector<DWARFAddressRange> Ranges; + + /// Sorted DWARFAddressRangeInfo. + std::set<DieRangeInfo> Children; + + DieRangeInfo() = default; + DieRangeInfo(DWARFDie Die) : Die(Die) {} + + /// Used for unit testing. + DieRangeInfo(std::vector<DWARFAddressRange> Ranges) + : Ranges(std::move(Ranges)) {} + + typedef std::vector<DWARFAddressRange>::const_iterator + address_range_iterator; + typedef std::set<DieRangeInfo>::const_iterator die_range_info_iterator; + + /// Inserts the address range. If the range overlaps with an existing + /// range, the range is *not* added and an iterator to the overlapping + /// range is returned. + /// + /// This is used for finding overlapping ranges within the same DIE. + address_range_iterator insert(const DWARFAddressRange &R); + + /// Finds an address range in the sorted vector of ranges. + address_range_iterator findRange(const DWARFAddressRange &R) const { + auto Begin = Ranges.begin(); + auto End = Ranges.end(); + auto Iter = std::upper_bound(Begin, End, R); + if (Iter != Begin) + --Iter; + return Iter; + } + + /// Inserts the address range info. If any of its ranges overlaps with a + /// range in an existing range info, the range info is *not* added and an + /// iterator to the overlapping range info. + /// + /// This is used for finding overlapping children of the same DIE. + die_range_info_iterator insert(const DieRangeInfo &RI); + + /// Return true if ranges in this object contains all ranges within RHS. + bool contains(const DieRangeInfo &RHS) const; + + /// Return true if any range in this object intersects with any range in + /// RHS. + bool intersects(const DieRangeInfo &RHS) const; + }; + +private: raw_ostream &OS; DWARFContext &DCtx; + DIDumpOptions DumpOpts; /// A map that tracks all references (converted absolute references) so we /// can verify each reference points to a valid DIE and not an offset that /// lies between to valid DIEs. std::map<uint64_t, std::set<uint32_t>> ReferenceToDIEOffsets; uint32_t NumDebugLineErrors = 0; - uint32_t NumAppleNamesErrors = 0; + + raw_ostream &error() const; + raw_ostream &warn() const; + raw_ostream ¬e() const; + + /// Verifies the abbreviations section. + /// + /// This function currently checks that: + /// --No abbreviation declaration has more than one attributes with the same + /// name. + /// + /// \param Abbrev Pointer to the abbreviations section we are verifying + /// Abbrev can be a pointer to either .debug_abbrev or debug_abbrev.dwo. + /// + /// \returns The number of errors that occured during verification. + unsigned verifyAbbrevSection(const DWARFDebugAbbrev *Abbrev); /// Verifies the header of a unit in the .debug_info section. /// @@ -58,8 +136,31 @@ class DWARFVerifier { uint32_t *Offset, unsigned UnitIndex, uint8_t &UnitType, bool &isUnitDWARF64); + /// Verifies the header of a unit in the .debug_info section. + /// + /// This function currently verifies: + /// - The debug info attributes. + /// - The debug info form=s. + /// - The presence of a root DIE. + /// - That the root DIE is a unit DIE. + /// - If a unit type is provided, that the unit DIE matches the unit type. + /// - The DIE ranges. + /// + /// \param Unit The DWARF Unit to verifiy. + /// \param UnitType An optional unit type which will be used to verify the + /// type of the unit DIE. + /// + /// \returns true if the content is verified successfully, false otherwise. + bool verifyUnitContents(DWARFUnit Unit, uint8_t UnitType = 0); + + /// Verify that all Die ranges are valid. + /// + /// This function currently checks for: + /// - cases in which lowPC >= highPC + /// + /// \returns Number of errors that occured during verification. + unsigned verifyDieRanges(const DWARFDie &Die, DieRangeInfo &ParentRI); - bool verifyUnitContents(DWARFUnit Unit); /// Verifies the attribute's DWARF attribute and its value. /// /// This function currently checks for: @@ -111,9 +212,40 @@ class DWARFVerifier { /// - invalid file indexes void verifyDebugLineRows(); + /// Verify that an Apple-style accelerator table is valid. + /// + /// This function currently checks that: + /// - The fixed part of the header fits in the section + /// - The size of the section is as large as what the header describes + /// - There is at least one atom + /// - The form for each atom is valid + /// - The tag for each DIE in the table is valid + /// - The buckets have a valid index, or they are empty + /// - Each hashdata offset is valid + /// - Each DIE is valid + /// + /// \param AccelSection pointer to the section containing the acceleration table + /// \param StrData pointer to the string section + /// \param SectionName the name of the table we're verifying + /// + /// \returns The number of errors occured during verification + unsigned verifyAccelTable(const DWARFSection *AccelSection, + DataExtractor *StrData, const char *SectionName); + public: - DWARFVerifier(raw_ostream &S, DWARFContext &D) - : OS(S), DCtx(D) {} + DWARFVerifier(raw_ostream &S, DWARFContext &D, + DIDumpOptions DumpOpts = DIDumpOptions::getForSingleDIE()) + : OS(S), DCtx(D), DumpOpts(std::move(DumpOpts)) {} + /// Verify the information in any of the following sections, if available: + /// .debug_abbrev, debug_abbrev.dwo + /// + /// Any errors are reported to the stream that was this object was + /// constructed with. + /// + /// \returns true if .debug_abbrev and .debug_abbrev.dwo verify successfully, + /// false otherwise. + bool handleDebugAbbrev(); + /// Verify the information in the .debug_info section. /// /// Any errors are reported to the stream that was this object was @@ -130,15 +262,21 @@ public: /// \returns true if the .debug_line verifies successfully, false otherwise. bool handleDebugLine(); - /// Verify the information in the .apple_names accelerator table. + /// Verify the information in accelerator tables, if they exist. /// /// Any errors are reported to the stream that was this object was /// constructed with. /// - /// \returns true if the .apple_names verifies successfully, false otherwise. - bool handleAppleNames(); + /// \returns true if the existing Apple-style accelerator tables verify + /// successfully, false otherwise. + bool handleAccelTables(); }; +static inline bool operator<(const DWARFVerifier::DieRangeInfo &LHS, + const DWARFVerifier::DieRangeInfo &RHS) { + return std::tie(LHS.Ranges, LHS.Die) < std::tie(RHS.Ranges, RHS.Die); +} + } // end namespace llvm #endif // LLVM_DEBUGINFO_DWARF_DWARFCONTEXT_H diff --git a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h index b2c8f2d1c20d..19e5c31b3076 100644 --- a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h @@ -128,7 +128,6 @@ private: uint32_t FreePageMap; uint32_t Unknown1 = 0; uint32_t BlockSize; - uint32_t MininumBlocks; uint32_t BlockMapAddr; BitVector FreeBlocks; std::vector<uint32_t> DirectoryBlocks; diff --git a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h index eca1b8b89ebd..f28415d4e603 100644 --- a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h +++ b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h @@ -59,6 +59,25 @@ struct MSFLayout { std::vector<ArrayRef<support::ulittle32_t>> StreamMap; }; +/// \brief Describes the layout of a stream in an MSF layout. A "stream" here +/// is defined as any logical unit of data which may be arranged inside the MSF +/// file as a sequence of (possibly discontiguous) blocks. When we want to read +/// from a particular MSF Stream, we fill out a stream layout structure and the +/// reader uses it to determine which blocks in the underlying MSF file contain +/// the data, so that it can be pieced together in the right order. +class MSFStreamLayout { +public: + uint32_t Length; + std::vector<support::ulittle32_t> Blocks; +}; + +/// \brief Determine the layout of the FPM stream, given the MSF layout. An FPM +/// stream spans 1 or more blocks, each at equally spaced intervals throughout +/// the file. +MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf, + bool IncludeUnusedFpmData = false, + bool AltFpm = false); + inline bool isValidBlockSize(uint32_t Size) { switch (Size) { case 512: @@ -78,7 +97,7 @@ inline uint32_t getMinimumBlockCount() { return 4; } inline uint32_t getFirstUnreservedBlock() { return 3; } inline uint64_t bytesToBlocks(uint64_t NumBytes, uint64_t BlockSize) { - return alignTo(NumBytes, BlockSize) / BlockSize; + return divideCeil(NumBytes, BlockSize); } inline uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize) { @@ -89,13 +108,14 @@ inline uint32_t getFpmIntervalLength(const MSFLayout &L) { return L.SB->BlockSize; } -inline uint32_t getNumFpmIntervals(const MSFLayout &L) { - uint32_t Length = getFpmIntervalLength(L); - return alignTo(L.SB->NumBlocks, Length) / Length; -} +inline uint32_t getNumFpmIntervals(const MSFLayout &L, + bool IncludeUnusedFpmData = false) { + if (IncludeUnusedFpmData) + return divideCeil(L.SB->NumBlocks, L.SB->BlockSize); -inline uint32_t getFullFpmByteSize(const MSFLayout &L) { - return alignTo(L.SB->NumBlocks, 8) / 8; + // We want the minimum number of intervals required, where each interval can + // represent BlockSize * 8 blocks. + return divideCeil(L.SB->NumBlocks, 8 * L.SB->BlockSize); } Error validateSuperBlock(const SuperBlock &SB); diff --git a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFStreamLayout.h b/contrib/llvm/include/llvm/DebugInfo/MSF/MSFStreamLayout.h deleted file mode 100644 index bdde98f52662..000000000000 --- a/contrib/llvm/include/llvm/DebugInfo/MSF/MSFStreamLayout.h +++ /dev/null @@ -1,35 +0,0 @@ -//===- MSFStreamLayout.h - Describes the layout of a stream -----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H -#define LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H - -#include "llvm/Support/Endian.h" - -#include <cstdint> -#include <vector> - -namespace llvm { -namespace msf { - -/// \brief Describes the layout of a stream in an MSF layout. A "stream" here -/// is defined as any logical unit of data which may be arranged inside the MSF -/// file as a sequence of (possibly discontiguous) blocks. When we want to read -/// from a particular MSF Stream, we fill out a stream layout structure and the -/// reader uses it to determine which blocks in the underlying MSF file contain -/// the data, so that it can be pieced together in the right order. -class MSFStreamLayout { -public: - uint32_t Length; - std::vector<support::ulittle32_t> Blocks; -}; -} // namespace msf -} // namespace llvm - -#endif // LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H diff --git a/contrib/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/contrib/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h index 6d88d2be85c9..f65e52922da7 100644 --- a/contrib/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h @@ -12,7 +12,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/DebugInfo/MSF/MSFStreamLayout.h" +#include "llvm/DebugInfo/MSF/MSFCommon.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryStream.h" #include "llvm/Support/BinaryStreamRef.h" @@ -122,7 +122,7 @@ public: static std::unique_ptr<WritableMappedBlockStream> createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData, - BumpPtrAllocator &Allocator); + BumpPtrAllocator &Allocator, bool AltFpm = false); support::endianness getEndian() const override { return support::little; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h new file mode 100644 index 000000000000..926fcfe69648 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h @@ -0,0 +1,37 @@ +//===- DIAEnumTables.h - DIA Tables Enumerator Impl -------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_DIA_DIAENUMTABLES_H +#define LLVM_DEBUGINFO_PDB_DIA_DIAENUMTABLES_H + +#include "DIASupport.h" +#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/IPDBTable.h" + +namespace llvm { +namespace pdb { +class IPDBTable; + +class DIAEnumTables : public IPDBEnumChildren<IPDBTable> { +public: + explicit DIAEnumTables(CComPtr<IDiaEnumTables> DiaEnumerator); + + uint32_t getChildCount() const override; + std::unique_ptr<IPDBTable> getChildAtIndex(uint32_t Index) const override; + std::unique_ptr<IPDBTable> getNext() override; + void reset() override; + DIAEnumTables *clone() const override; + +private: + CComPtr<IDiaEnumTables> Enumerator; +}; +} +} + +#endif // LLVM_DEBUGINFO_PDB_DIA_DIAENUMTABLES_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h index d37b48540ffa..2d6c44905ce0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h @@ -96,6 +96,7 @@ public: uint32_t getTypeId() const override; uint32_t getUavSlot() const override; std::string getUndecoratedName() const override; + std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const override; uint32_t getUnmodifiedTypeId() const override; uint32_t getUpperBoundId() const override; Variant getValue() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h index 350442556bef..66bd7a7e9c4e 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIASession.h @@ -64,6 +64,7 @@ public: std::unique_ptr<IPDBEnumDataStreams> getDebugStreams() const override; + std::unique_ptr<IPDBEnumTables> getEnumTables() const override; private: CComPtr<IDiaSession> Session; }; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIATable.h b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIATable.h new file mode 100644 index 000000000000..ce93fa0b86c3 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/DIA/DIATable.h @@ -0,0 +1,32 @@ +//===- DIATable.h - DIA implementation of IPDBTable -------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_DIA_DIATABLE_H +#define LLVM_DEBUGINFO_PDB_DIA_DIATABLE_H + +#include "DIASupport.h" +#include "llvm/DebugInfo/PDB/IPDBTable.h" + +namespace llvm { +namespace pdb { +class DIATable : public IPDBTable { +public: + explicit DIATable(CComPtr<IDiaTable> DiaTable); + + uint32_t getItemCount() const override; + std::string getName() const override; + PDB_TableType getTableType() const override; + +private: + CComPtr<IDiaTable> Table; +}; +} +} + +#endif // LLVM_DEBUGINFO_PDB_DIA_DIATABLE_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h index eefc36518728..18b9423378a0 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h @@ -108,6 +108,7 @@ public: virtual uint32_t getTypeId() const = 0; virtual uint32_t getUavSlot() const = 0; virtual std::string getUndecoratedName() const = 0; + virtual std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const = 0; virtual uint32_t getUnmodifiedTypeId() const = 0; virtual uint32_t getUpperBoundId() const = 0; virtual Variant getValue() const = 0; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h index cf195095c8d2..6291289de5bf 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBSession.h @@ -67,6 +67,8 @@ public: getSourceFileById(uint32_t FileId) const = 0; virtual std::unique_ptr<IPDBEnumDataStreams> getDebugStreams() const = 0; + + virtual std::unique_ptr<IPDBEnumTables> getEnumTables() const = 0; }; } } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBTable.h b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBTable.h new file mode 100644 index 000000000000..4561c4e847b2 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/IPDBTable.h @@ -0,0 +1,28 @@ +//===- IPDBTable.h - Base Interface for a PDB Symbol Context ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_IPDBTABLE_H +#define LLVM_DEBUGINFO_PDB_IPDBTABLE_H + +#include "PDBTypes.h" + +namespace llvm { +namespace pdb { +class IPDBTable { +public: + virtual ~IPDBTable(); + + virtual std::string getName() const = 0; + virtual uint32_t getItemCount() const = 0; + virtual PDB_TableType getTableType() const = 0; +}; +} +} + +#endif // LLVM_DEBUGINFO_PDB_IPDBTABLE_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index 63eb34f0326a..ad4a0d1bcb6b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -59,11 +59,11 @@ public: uint32_t calculateSerializedLength() const; + void setGlobalsStreamIndex(uint32_t Index); void setPublicsStreamIndex(uint32_t Index); void setSymbolRecordStreamIndex(uint32_t Index); Expected<DbiModuleDescriptorBuilder &> addModuleInfo(StringRef ModuleName); - Error addModuleSourceFile(StringRef Module, StringRef File); Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File); Expected<uint32_t> getSourceFileNameIndex(StringRef FileName); @@ -71,8 +71,9 @@ public: Error commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef MsfBuffer); - void addSectionContrib(DbiModuleDescriptorBuilder *ModuleDbi, - const llvm::object::coff_section *SecHdr); + void addSectionContrib(const SectionContrib &SC) { + SectionContribs.emplace_back(SC); + } // A helper function to create a Section Map from a COFF section header. static std::vector<SecMapEntry> @@ -105,13 +106,13 @@ private: uint16_t PdbDllRbld; uint16_t Flags; PDB_Machine MachineType; + uint32_t GlobalsStreamIndex = kInvalidStreamIndex; uint32_t PublicsStreamIndex = kInvalidStreamIndex; uint32_t SymRecordStreamIndex = kInvalidStreamIndex; const DbiStreamHeader *Header; - StringMap<std::unique_ptr<DbiModuleDescriptorBuilder>> ModiMap; - std::vector<DbiModuleDescriptorBuilder *> ModiList; + std::vector<std::unique_ptr<DbiModuleDescriptorBuilder>> ModiList; StringMap<uint32_t> SourceFileNames; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h new file mode 100644 index 000000000000..1a4f89d607df --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h @@ -0,0 +1,82 @@ +//===- GSIStreamBuilder.h - PDB Publics/Globals Stream Creation -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_RAW_GSISTREAMBUILDER_H +#define LLVM_DEBUGINFO_PDB_RAW_GSISTREAMBUILDER_H + +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/BinaryItemStream.h" +#include "llvm/Support/BinaryStreamRef.h" +#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" + +namespace llvm { + +template <> struct BinaryItemTraits<codeview::CVSymbol> { + static size_t length(const codeview::CVSymbol &Item) { + return Item.RecordData.size(); + } + static ArrayRef<uint8_t> bytes(const codeview::CVSymbol &Item) { + return Item.RecordData; + } +}; + +namespace msf { +class MSFBuilder; +struct MSFLayout; +} // namespace msf +namespace pdb { +struct GSIHashStreamBuilder; + +class GSIStreamBuilder { + +public: + explicit GSIStreamBuilder(msf::MSFBuilder &Msf); + ~GSIStreamBuilder(); + + GSIStreamBuilder(const GSIStreamBuilder &) = delete; + GSIStreamBuilder &operator=(const GSIStreamBuilder &) = delete; + + Error finalizeMsfLayout(); + + Error commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef Buffer); + + uint32_t getPublicsStreamIndex() const; + uint32_t getGlobalsStreamIndex() const; + uint32_t getRecordStreamIdx() const { return RecordStreamIdx; } + + void addPublicSymbol(const codeview::PublicSym32 &Pub); + + void addGlobalSymbol(const codeview::ProcRefSym &Sym); + void addGlobalSymbol(const codeview::DataSym &Sym); + void addGlobalSymbol(const codeview::ConstantSym &Sym); + void addGlobalSymbol(const codeview::UDTSym &Sym); + void addGlobalSymbol(const codeview::CVSymbol &Sym); + +private: + uint32_t calculatePublicsHashStreamSize() const; + uint32_t calculateGlobalsHashStreamSize() const; + Error commitSymbolRecordStream(WritableBinaryStreamRef Stream); + Error commitPublicsHashStream(WritableBinaryStreamRef Stream); + Error commitGlobalsHashStream(WritableBinaryStreamRef Stream); + + uint32_t RecordStreamIdx = kInvalidStreamIndex; + msf::MSFBuilder &Msf; + std::unique_ptr<GSIHashStreamBuilder> PSH; + std::unique_ptr<GSIHashStreamBuilder> GSH; +}; +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h index dcea3d3be0ab..fdc58dc60f7e 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h @@ -1,4 +1,4 @@ -//===- GlobalsStream.h - PDB Index of Symbols by Name ------ ----*- C++ -*-===// +//===- GlobalsStream.h - PDB Index of Symbols by Name -----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,27 +16,66 @@ #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/Error.h" +#include "llvm/ADT/iterator.h" namespace llvm { namespace pdb { class DbiStream; class PDBFile; +/// Iterator over hash records producing symbol record offsets. Abstracts away +/// the fact that symbol record offsets on disk are off-by-one. +class GSIHashIterator + : public iterator_adaptor_base< + GSIHashIterator, FixedStreamArrayIterator<PSHashRecord>, + std::random_access_iterator_tag, const uint32_t> { +public: + GSIHashIterator() = default; + + template <typename T> + GSIHashIterator(T &&v) + : GSIHashIterator::iterator_adaptor_base(std::forward<T &&>(v)) {} + + uint32_t operator*() const { + uint32_t Off = this->I->Off; + return --Off; + } +}; + +/// From https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.cpp +enum : unsigned { IPHR_HASH = 4096 }; + +/// A readonly view of a hash table used in the globals and publics streams. +/// Most clients will only want to iterate this to get symbol record offsets +/// into the PDB symbol stream. +class GSIHashTable { +public: + const GSIHashHeader *HashHdr; + FixedStreamArray<PSHashRecord> HashRecords; + ArrayRef<uint8_t> HashBitmap; + FixedStreamArray<support::ulittle32_t> HashBuckets; + + Error read(BinaryStreamReader &Reader); + + uint32_t getVerSignature() const { return HashHdr->VerSignature; } + uint32_t getVerHeader() const { return HashHdr->VerHdr; } + uint32_t getHashRecordSize() const { return HashHdr->HrSize; } + uint32_t getNumBuckets() const { return HashHdr->NumBuckets; } + + typedef GSIHashHeader iterator; + GSIHashIterator begin() const { return GSIHashIterator(HashRecords.begin()); } + GSIHashIterator end() const { return GSIHashIterator(HashRecords.end()); } +}; + class GlobalsStream { public: explicit GlobalsStream(std::unique_ptr<msf::MappedBlockStream> Stream); ~GlobalsStream(); - Error commit(); - FixedStreamArray<support::ulittle32_t> getHashBuckets() const { - return HashBuckets; - } - uint32_t getNumBuckets() const { return NumBuckets; } + const GSIHashTable &getGlobalsTable() const { return GlobalsTable; } Error reload(); private: - FixedStreamArray<support::ulittle32_t> HashBuckets; - FixedStreamArray<PSHashRecord> HashRecords; - uint32_t NumBuckets; + GSIHashTable GlobalsTable; std::unique_ptr<msf::MappedBlockStream> Stream; }; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h index f413fd1b336e..6602264d1b74 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h @@ -32,6 +32,7 @@ public: ModuleDebugStreamRef(const DbiModuleDescriptor &Module, std::unique_ptr<msf::MappedBlockStream> Stream); ModuleDebugStreamRef(ModuleDebugStreamRef &&Other) = default; + ModuleDebugStreamRef(const ModuleDebugStreamRef &Other) = default; ~ModuleDebugStreamRef(); Error reload(); @@ -51,6 +52,9 @@ public: ModuleDebugStreamRef &operator=(ModuleDebugStreamRef &&Other) = default; iterator_range<DebugSubsectionIterator> subsections() const; + codeview::DebugSubsectionArray getSubsectionsArray() const { + return Subsections; + } bool hasDebugSubsections() const; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h new file mode 100644 index 000000000000..41b7b78b8d80 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumSymbol.h @@ -0,0 +1,60 @@ +//===- NativeEnumSymbol.h - info about enum type ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/DebugInfo/PDB/Native/NativeRawSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" + +namespace llvm { +namespace pdb { + +class NativeEnumSymbol : public NativeRawSymbol, + public codeview::TypeVisitorCallbacks { +public: + NativeEnumSymbol(NativeSession &Session, SymIndexId Id, + const codeview::CVType &CV); + ~NativeEnumSymbol() override; + + std::unique_ptr<NativeRawSymbol> clone() const override; + + std::unique_ptr<IPDBEnumSymbols> + findChildren(PDB_SymType Type) const override; + + Error visitKnownRecord(codeview::CVType &CVR, + codeview::EnumRecord &Record) override; + Error visitKnownMember(codeview::CVMemberRecord &CVM, + codeview::EnumeratorRecord &Record) override; + + PDB_SymType getSymTag() const override; + uint32_t getClassParentId() const override; + uint32_t getUnmodifiedTypeId() const override; + bool hasConstructor() const override; + bool hasAssignmentOperator() const override; + bool hasCastOperator() const override; + uint64_t getLength() const override; + std::string getName() const override; + bool isNested() const override; + bool hasOverloadedOperator() const override; + bool isPacked() const override; + bool isScoped() const override; + uint32_t getTypeId() const override; + +protected: + codeview::CVType CV; + codeview::EnumRecord Record; +}; + +} // namespace pdb +} // namespace llvm + +#endif // LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMSYMBOL_H diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h new file mode 100644 index 000000000000..e0a5c8d9ad81 --- /dev/null +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h @@ -0,0 +1,51 @@ +//==- NativeEnumTypes.h - Native Type Enumerator impl ------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMTYPES_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMTYPES_H + +#include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/PDBSymbol.h" + +#include <vector> + +namespace llvm { +namespace pdb { + +class NativeSession; + +class NativeEnumTypes : public IPDBEnumChildren<PDBSymbol> { +public: + NativeEnumTypes(NativeSession &Session, + codeview::LazyRandomTypeCollection &TypeCollection, + codeview::TypeLeafKind Kind); + + uint32_t getChildCount() const override; + std::unique_ptr<PDBSymbol> getChildAtIndex(uint32_t Index) const override; + std::unique_ptr<PDBSymbol> getNext() override; + void reset() override; + NativeEnumTypes *clone() const override; + +private: + NativeEnumTypes(NativeSession &Session, + const std::vector<codeview::TypeIndex> &Matches, + codeview::TypeLeafKind Kind); + + std::vector<codeview::TypeIndex> Matches; + uint32_t Index; + NativeSession &Session; + codeview::TypeLeafKind Kind; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h index 2c6548dcce21..931b93fb7266 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h @@ -101,6 +101,7 @@ public: uint32_t getTypeId() const override; uint32_t getUavSlot() const override; std::string getUndecoratedName() const override; + std::string getUndecoratedNameEx(PDB_UndnameFlags Flags) const override; uint32_t getUnmodifiedTypeId() const override; uint32_t getUpperBoundId() const override; Variant getValue() const override; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h index b16ce231c349..2e68ced46bfe 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/NativeSession.h @@ -22,6 +22,7 @@ #include "llvm/Support/Error.h" namespace llvm { +class MemoryBuffer; namespace pdb { class PDBFile; @@ -31,7 +32,7 @@ public: std::unique_ptr<BumpPtrAllocator> Allocator); ~NativeSession() override; - static Error createFromPdb(StringRef Path, + static Error createFromPdb(std::unique_ptr<MemoryBuffer> MB, std::unique_ptr<IPDBSession> &Session); static Error createFromExe(StringRef Path, std::unique_ptr<IPDBSession> &Session); @@ -39,6 +40,12 @@ public: std::unique_ptr<PDBSymbolCompiland> createCompilandSymbol(DbiModuleDescriptor MI); + std::unique_ptr<PDBSymbolTypeEnum> + createEnumSymbol(codeview::TypeIndex Index); + + std::unique_ptr<IPDBEnumSymbols> + createTypeEnumerator(codeview::TypeLeafKind Kind); + SymIndexId findSymbolByTypeIndex(codeview::TypeIndex TI); uint64_t getLoadAddress() const override; @@ -76,6 +83,8 @@ public: std::unique_ptr<IPDBEnumDataStreams> getDebugStreams() const override; + std::unique_ptr<IPDBEnumTables> getEnumTables() const override; + PDBFile &getPDBFile() { return *Pdb; } const PDBFile &getPDBFile() const { return *Pdb; } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h index 4f6ad115e7df..5e39ac3e37b7 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h @@ -13,7 +13,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/DebugInfo/MSF/IMSFFile.h" #include "llvm/DebugInfo/MSF/MSFCommon.h" -#include "llvm/DebugInfo/MSF/MSFStreamLayout.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryStreamRef.h" #include "llvm/Support/Endian.h" @@ -62,6 +61,7 @@ public: uint64_t getBlockMapOffset() const; uint32_t getNumStreams() const override; + uint32_t getMaxStreamSize() const; uint32_t getStreamByteSize(uint32_t StreamIndex) const override; ArrayRef<support::ulittle32_t> getStreamBlockList(uint32_t StreamIndex) const override; @@ -72,8 +72,6 @@ public: Error setBlockData(uint32_t BlockIndex, uint32_t Offset, ArrayRef<uint8_t> Data) const override; - ArrayRef<uint32_t> getFpmPages() const { return FpmPages; } - ArrayRef<support::ulittle32_t> getStreamSizes() const { return ContainerLayout.StreamSizes; } @@ -86,7 +84,10 @@ public: ArrayRef<support::ulittle32_t> getDirectoryBlockArray() const; + std::unique_ptr<msf::MappedBlockStream> createIndexedStream(uint16_t SN); + msf::MSFStreamLayout getStreamLayout(uint32_t StreamIdx) const; + msf::MSFStreamLayout getFpmStreamLayout() const; Error parseFileHeaders(); Error parseStreamData(); @@ -104,7 +105,7 @@ public: bool hasPDBDbiStream() const; bool hasPDBGlobalsStream(); - bool hasPDBInfoStream(); + bool hasPDBInfoStream() const; bool hasPDBIpiStream() const; bool hasPDBPublicsStream(); bool hasPDBSymbolStream(); @@ -124,7 +125,6 @@ private: std::unique_ptr<BinaryStream> Buffer; - std::vector<uint32_t> FpmPages; msf::MSFLayout ContainerLayout; std::unique_ptr<GlobalsStream> Globals; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index 2dc23f819d3b..7ed164bee9ee 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -31,7 +31,7 @@ class MSFBuilder; namespace pdb { class DbiStreamBuilder; class InfoStreamBuilder; -class PublicsStreamBuilder; +class GSIStreamBuilder; class TpiStreamBuilder; class PDBFileBuilder { @@ -49,7 +49,7 @@ public: TpiStreamBuilder &getTpiBuilder(); TpiStreamBuilder &getIpiBuilder(); PDBStringTableBuilder &getStringTableBuilder(); - PublicsStreamBuilder &getPublicsBuilder(); + GSIStreamBuilder &getGsiBuilder(); Error commit(StringRef Filename); @@ -59,12 +59,14 @@ public: private: Expected<msf::MSFLayout> finalizeMsfLayout(); + void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout); + BumpPtrAllocator &Allocator; std::unique_ptr<msf::MSFBuilder> Msf; std::unique_ptr<InfoStreamBuilder> Info; std::unique_ptr<DbiStreamBuilder> Dbi; - std::unique_ptr<PublicsStreamBuilder> Publics; + std::unique_ptr<GSIStreamBuilder> Gsi; std::unique_ptr<TpiStreamBuilder> Tpi; std::unique_ptr<TpiStreamBuilder> Ipi; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h index 9ace826bd8f7..2d0222a9071a 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStream.h @@ -12,6 +12,7 @@ #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" +#include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" @@ -26,19 +27,14 @@ class PDBFile; class PublicsStream { public: - PublicsStream(PDBFile &File, std::unique_ptr<msf::MappedBlockStream> Stream); + PublicsStream(std::unique_ptr<msf::MappedBlockStream> Stream); ~PublicsStream(); Error reload(); uint32_t getSymHash() const; - uint32_t getAddrMap() const; - uint32_t getNumBuckets() const { return NumBuckets; } - Expected<const codeview::CVSymbolArray &> getSymbolArray() const; - iterator_range<codeview::CVSymbolArray::Iterator> - getSymbols(bool *HadError) const; - FixedStreamArray<support::ulittle32_t> getHashBuckets() const { - return HashBuckets; - } + uint16_t getThunkTableSection() const; + uint32_t getThunkTableOffset() const; + const GSIHashTable &getPublicsTable() const { return PublicsTable; } FixedStreamArray<support::ulittle32_t> getAddressMap() const { return AddressMap; } @@ -49,22 +45,14 @@ public: return SectionOffsets; } - Error commit(); - private: - PDBFile &Pdb; - std::unique_ptr<msf::MappedBlockStream> Stream; - uint32_t NumBuckets = 0; - ArrayRef<uint8_t> Bitmap; - FixedStreamArray<PSHashRecord> HashRecords; - FixedStreamArray<support::ulittle32_t> HashBuckets; + GSIHashTable PublicsTable; FixedStreamArray<support::ulittle32_t> AddressMap; FixedStreamArray<support::ulittle32_t> ThunkMap; FixedStreamArray<SectionOffset> SectionOffsets; const PublicsStreamHeader *Header; - const GSIHashHeader *HashHdr; }; } } diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h deleted file mode 100644 index 5ab57ebef53d..000000000000 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h +++ /dev/null @@ -1,54 +0,0 @@ -//===- PublicsStreamBuilder.h - PDB Publics Stream Creation -----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H -#define LLVM_DEBUGINFO_PDB_RAW_PDBPUBLICSTREAMBUILDER_H - -#include "llvm/DebugInfo/PDB/Native/RawConstants.h" -#include "llvm/DebugInfo/PDB/Native/RawTypes.h" -#include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/BinaryStreamRef.h" -#include "llvm/Support/BinaryStreamWriter.h" -#include "llvm/Support/Endian.h" -#include "llvm/Support/Error.h" - -namespace llvm { -namespace msf { -class MSFBuilder; -} -namespace pdb { -class PublicsStream; -struct PublicsStreamHeader; - -class PublicsStreamBuilder { -public: - explicit PublicsStreamBuilder(msf::MSFBuilder &Msf); - ~PublicsStreamBuilder(); - - PublicsStreamBuilder(const PublicsStreamBuilder &) = delete; - PublicsStreamBuilder &operator=(const PublicsStreamBuilder &) = delete; - - Error finalizeMsfLayout(); - uint32_t calculateSerializedLength() const; - - Error commit(BinaryStreamWriter &PublicsWriter); - - uint32_t getStreamIndex() const { return StreamIdx; } - uint32_t getRecordStreamIdx() const { return RecordStreamIdx; } - -private: - uint32_t StreamIdx = kInvalidStreamIndex; - uint32_t RecordStreamIdx = kInvalidStreamIndex; - std::vector<PSHashRecord> HashRecords; - msf::MSFBuilder &Msf; -}; -} // namespace pdb -} // namespace llvm - -#endif diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h index b6321cbf45a8..8cc083685265 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -23,6 +23,20 @@ struct SectionOffset { char Padding[2]; }; +/// Header of the hash tables found in the globals and publics sections. +/// Based on GSIHashHdr in +/// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h +struct GSIHashHeader { + enum : unsigned { + HdrSignature = ~0U, + HdrVersion = 0xeffe0000 + 19990810, + }; + support::ulittle32_t VerSignature; + support::ulittle32_t VerHdr; + support::ulittle32_t HrSize; + support::ulittle32_t NumBuckets; +}; + // This is HRFile. struct PSHashRecord { support::ulittle32_t Off; // Offset in the symbol record stream diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h index 17695f587849..ae9f7d657b70 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/Native/SymbolStream.h @@ -31,6 +31,8 @@ public: return SymbolRecords; } + codeview::CVSymbol readRecord(uint32_t Offset) const; + iterator_range<codeview::CVSymbolArray::Iterator> getSymbols(bool *HadError) const; diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h index 9e883d2f99a7..04373463212b 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEBUGINFO_PDB_IPDBSYMBOL_H -#define LLVM_DEBUGINFO_PDB_IPDBSYMBOL_H +#ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOL_H +#define LLVM_DEBUGINFO_PDB_PDBSYMBOL_H #include "ConcreteSymbolEnumerator.h" #include "IPDBRawSymbol.h" diff --git a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h index 79ec7ce906d5..a6c6da37d1cc 100644 --- a/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h +++ b/contrib/llvm/include/llvm/DebugInfo/PDB/PDBTypes.h @@ -13,6 +13,7 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include <cctype> #include <cstddef> #include <cstdint> #include <cstring> @@ -24,6 +25,7 @@ namespace pdb { class IPDBDataStream; class IPDBLineNumber; class IPDBSourceFile; +class IPDBTable; class PDBSymDumper; class PDBSymbol; class PDBSymbolExe; @@ -62,6 +64,7 @@ using IPDBEnumSymbols = IPDBEnumChildren<PDBSymbol>; using IPDBEnumSourceFiles = IPDBEnumChildren<IPDBSourceFile>; using IPDBEnumDataStreams = IPDBEnumChildren<IPDBDataStream>; using IPDBEnumLineNumbers = IPDBEnumChildren<IPDBLineNumber>; +using IPDBEnumTables = IPDBEnumChildren<IPDBTable>; /// Specifies which PDB reader implementation is to be used. Only a value /// of PDB_ReaderType::DIA is currently supported, but Native is in the works. @@ -72,13 +75,16 @@ enum class PDB_ReaderType { /// An enumeration indicating the type of data contained in this table. enum class PDB_TableType { + TableInvalid = 0, Symbols, SourceFiles, LineNumbers, SectionContribs, Segments, InjectedSources, - FrameData + FrameData, + InputAssemblyFiles, + Dbg }; /// Defines flags used for enumerating child symbols. This corresponds to the @@ -241,6 +247,32 @@ enum class PDB_BuiltinType { HResult = 31 }; +/// These values correspond to the flags that can be combined to control the +/// return of an undecorated name for a C++ decorated name, and are documented +/// here: https://msdn.microsoft.com/en-us/library/kszfk0fs.aspx +enum PDB_UndnameFlags: uint32_t { + Undname_Complete = 0x0, + Undname_NoLeadingUnderscores = 0x1, + Undname_NoMsKeywords = 0x2, + Undname_NoFuncReturns = 0x4, + Undname_NoAllocModel = 0x8, + Undname_NoAllocLang = 0x10, + Undname_Reserved1 = 0x20, + Undname_Reserved2 = 0x40, + Undname_NoThisType = 0x60, + Undname_NoAccessSpec = 0x80, + Undname_NoThrowSig = 0x100, + Undname_NoMemberType = 0x200, + Undname_NoReturnUDTModel = 0x400, + Undname_32BitDecode = 0x800, + Undname_NameOnly = 0x1000, + Undname_TypeOnly = 0x2000, + Undname_HaveParams = 0x4000, + Undname_NoECSU = 0x8000, + Undname_NoIdentCharCheck = 0x10000, + Undname_NoPTR64 = 0x20000 +}; + enum class PDB_MemberAccess { Private = 1, Protected = 2, Public = 3 }; struct VersionInfo { diff --git a/contrib/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/contrib/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h index d98d49b24bca..6480aef109c6 100644 --- a/contrib/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/contrib/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -58,9 +58,11 @@ public: } Expected<DILineInfo> symbolizeCode(const std::string &ModuleName, - uint64_t ModuleOffset); + uint64_t ModuleOffset, + StringRef DWPName = ""); Expected<DIInliningInfo> symbolizeInlinedCode(const std::string &ModuleName, - uint64_t ModuleOffset); + uint64_t ModuleOffset, + StringRef DWPName = ""); Expected<DIGlobal> symbolizeData(const std::string &ModuleName, uint64_t ModuleOffset); void flush(); @@ -79,7 +81,7 @@ private: /// only reported once. Subsequent calls to get module info for a module that /// failed to load will return nullptr. Expected<SymbolizableModule *> - getOrCreateModuleInfo(const std::string &ModuleName); + getOrCreateModuleInfo(const std::string &ModuleName, StringRef DWPName = ""); ObjectFile *lookUpDsymFile(const std::string &Path, const MachOObjectFile *ExeObj, diff --git a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h index 2830a2628753..77c23b46d320 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -535,12 +535,13 @@ private: std::shared_ptr<JITSymbolResolver> Resolver; TargetOptions Options; Optional<Reloc::Model> RelocModel; - CodeModel::Model CMModel; + Optional<CodeModel::Model> CMModel; std::string MArch; std::string MCPU; SmallVector<std::string, 4> MAttrs; bool VerifyModules; bool UseOrcMCJITReplacement; + bool EmulatedTLS = true; public: /// Default constructor for EngineBuilder. @@ -641,6 +642,10 @@ public: this->UseOrcMCJITReplacement = UseOrcMCJITReplacement; } + void setEmulatedTLS(bool EmulatedTLS) { + this->EmulatedTLS = EmulatedTLS; + } + TargetMachine *selectTarget(); /// selectTarget - Pick a target either via -march or by guessing the native diff --git a/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h index 4172f240ba39..933b3ea8e13d 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/JITSymbol.h @@ -40,6 +40,7 @@ using JITTargetAddress = uint64_t; class JITSymbolFlags { public: using UnderlyingType = uint8_t; + using TargetFlagsType = uint64_t; enum FlagNames : UnderlyingType { None = 0, @@ -56,32 +57,48 @@ public: /// @brief Construct a JITSymbolFlags instance from the given flags. JITSymbolFlags(FlagNames Flags) : Flags(Flags) {} + /// @brief Construct a JITSymbolFlags instance from the given flags and target + /// flags. + JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) + : Flags(Flags), TargetFlags(TargetFlags) {} + /// @brief Return true if there was an error retrieving this symbol. bool hasError() const { return (Flags & HasError) == HasError; } - /// @brief Returns true is the Weak flag is set. + /// @brief Returns true if the Weak flag is set. bool isWeak() const { return (Flags & Weak) == Weak; } - /// @brief Returns true is the Weak flag is set. + /// @brief Returns true if the Common flag is set. bool isCommon() const { return (Flags & Common) == Common; } + /// @brief Returns true if the symbol isn't weak or common. bool isStrongDefinition() const { return !isWeak() && !isCommon(); } - /// @brief Returns true is the Weak flag is set. + /// @brief Returns true if the Exported flag is set. bool isExported() const { return (Flags & Exported) == Exported; } + /// @brief Implicitly convert to the underlying flags type. operator UnderlyingType&() { return Flags; } + /// @brief Implicitly convert to the underlying flags type. + operator const UnderlyingType&() const { return Flags; } + + /// @brief Return a reference to the target-specific flags. + TargetFlagsType& getTargetFlags() { return TargetFlags; } + + /// @brief Return a reference to the target-specific flags. + const TargetFlagsType& getTargetFlags() const { return TargetFlags; } + /// Construct a JITSymbolFlags value based on the flags of the given global /// value. static JITSymbolFlags fromGlobalValue(const GlobalValue &GV); @@ -92,6 +109,26 @@ public: private: UnderlyingType Flags = None; + TargetFlagsType TargetFlags = 0; +}; + +/// @brief ARM-specific JIT symbol flags. +/// FIXME: This should be moved into a target-specific header. +class ARMJITSymbolFlags { +public: + ARMJITSymbolFlags() = default; + + enum FlagNames { + None = 0, + Thumb = 1 << 0 + }; + + operator JITSymbolFlags::TargetFlagsType&() { return Flags; } + + static ARMJITSymbolFlags fromObjectSymbol( + const object::BasicSymbolRef &Symbol); +private: + JITSymbolFlags::TargetFlagsType Flags = 0; }; /// @brief Represents a symbol that has been evaluated to an address already. diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 27b5457fc8ff..a961992c2147 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -140,12 +140,6 @@ private: struct LogicalDylib { using SymbolResolverFtor = std::function<JITSymbol(const std::string&)>; - using ModuleAdderFtor = - std::function<typename BaseLayerT::ModuleHandleT( - BaseLayerT&, - std::unique_ptr<Module>, - std::unique_ptr<JITSymbolResolver>)>; - struct SourceModuleEntry { std::shared_ptr<Module> SourceMod; std::set<Function*> StubsToClone; @@ -349,19 +343,22 @@ private: // Create a callback, associate it with the stub for the function, // and set the compile action to compile the partition containing the // function. - auto CCInfo = CompileCallbackMgr.getCompileCallback(); - StubInits[MangledName] = - std::make_pair(CCInfo.getAddress(), - JITSymbolFlags::fromGlobalValue(F)); - CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress { - if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) - return *FnImplAddrOrErr; - else { - // FIXME: Report error, return to 'abort' or something similar. - consumeError(FnImplAddrOrErr.takeError()); - return 0; - } - }); + if (auto CCInfoOrErr = CompileCallbackMgr.getCompileCallback()) { + auto &CCInfo = *CCInfoOrErr; + StubInits[MangledName] = + std::make_pair(CCInfo.getAddress(), + JITSymbolFlags::fromGlobalValue(F)); + CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress { + if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) + return *FnImplAddrOrErr; + else { + // FIXME: Report error, return to 'abort' or something similar. + consumeError(FnImplAddrOrErr.takeError()); + return 0; + } + }); + } else + return CCInfoOrErr.takeError(); } if (auto Err = LD.StubsMgr->createStubs(StubInits)) diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h index ff54ef625ebb..8a48c36f4141 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h @@ -46,13 +46,14 @@ public: /// @brief Add the given module to the JIT. /// @return A handle for the added modules. - ModuleHandleT addModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) { + Expected<ModuleHandleT> + addModule(std::shared_ptr<Module> M, + std::shared_ptr<JITSymbolResolver> Resolver) { return BaseLayer.addModule(std::move(M), std::move(Resolver)); } /// @brief Remove the module set associated with the handle H. - void removeModule(ModuleHandleT H) { BaseLayer.removeModule(H); } + Error removeModule(ModuleHandleT H) { return BaseLayer.removeModule(H); } /// @brief Manually set the address to return for the given symbol. void setGlobalMapping(const std::string &Name, JITTargetAddress Addr) { @@ -96,8 +97,8 @@ public: /// @brief Immediately emit and finalize the module set represented by the /// given handle. /// @param H Handle for module set to emit/finalize. - void emitAndFinalize(ModuleHandleT H) { - BaseLayer.emitAndFinalize(H); + Error emitAndFinalize(ModuleHandleT H) { + return BaseLayer.emitAndFinalize(H); } private: diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index e038093d7628..029b86a6d2ca 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -105,10 +105,13 @@ public: } /// @brief Reserve a compile callback. - CompileCallbackInfo getCompileCallback() { - JITTargetAddress TrampolineAddr = getAvailableTrampolineAddr(); - auto &Compile = this->ActiveTrampolines[TrampolineAddr]; - return CompileCallbackInfo(TrampolineAddr, Compile); + Expected<CompileCallbackInfo> getCompileCallback() { + if (auto TrampolineAddrOrErr = getAvailableTrampolineAddr()) { + const auto &TrampolineAddr = *TrampolineAddrOrErr; + auto &Compile = this->ActiveTrampolines[TrampolineAddr]; + return CompileCallbackInfo(TrampolineAddr, Compile); + } else + return TrampolineAddrOrErr.takeError(); } /// @brief Get a CompileCallbackInfo for an existing callback. @@ -138,9 +141,10 @@ protected: std::vector<JITTargetAddress> AvailableTrampolines; private: - JITTargetAddress getAvailableTrampolineAddr() { + Expected<JITTargetAddress> getAvailableTrampolineAddr() { if (this->AvailableTrampolines.empty()) - grow(); + if (auto Err = grow()) + return std::move(Err); assert(!this->AvailableTrampolines.empty() && "Failed to grow available trampolines."); JITTargetAddress TrampolineAddr = this->AvailableTrampolines.back(); @@ -149,7 +153,7 @@ private: } // Create new trampolines - to be implemented in subclasses. - virtual void grow() = 0; + virtual Error grow() = 0; virtual void anchor(); }; @@ -188,7 +192,7 @@ private: reinterpret_cast<uintptr_t>(TrampolineId))); } - void grow() override { + Error grow() override { assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); std::error_code EC; @@ -196,7 +200,8 @@ private: sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( sys::Process::getPageSize(), nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - assert(!EC && "Failed to allocate trampoline block"); + if (EC) + return errorCodeToError(EC); unsigned NumTrampolines = (sys::Process::getPageSize() - TargetT::PointerSize) / @@ -211,12 +216,13 @@ private: static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>( TrampolineMem + (I * TargetT::TrampolineSize)))); - EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - assert(!EC && "Failed to mprotect trampoline block"); + if (auto EC = sys::Memory::protectMappedMemory( + TrampolineBlock.getMemoryBlock(), + sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); TrampolineBlocks.push_back(std::move(TrampolineBlock)); + return Error::success(); } sys::OwningMemoryBlock ResolverBlock; diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h index e6374b70967a..e1ac87075ac0 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -33,7 +33,8 @@ enum class OrcErrorCode : int { RPCResponseAbandoned, UnexpectedRPCCall, UnexpectedRPCResponse, - UnknownErrorCodeFromRemote + UnknownErrorCodeFromRemote, + UnknownResourceHandle }; std::error_code orcError(OrcErrorCode ErrCode); diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index da02250ba169..7179e5ff66fd 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -53,27 +53,26 @@ namespace remote { /// Each of the utility classes talks to a JIT server (an instance of the /// OrcRemoteTargetServer class) via an RPC system (see RPCUtils.h) to carry out /// its actions. -template <typename ChannelT> -class OrcRemoteTargetClient : public OrcRemoteTargetRPCAPI { +class OrcRemoteTargetClient + : public rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel> { public: - /// Remote memory manager. - class RCMemoryManager : public RuntimeDyld::MemoryManager { - public: - RCMemoryManager(OrcRemoteTargetClient &Client, ResourceIdMgr::ResourceId Id) - : Client(Client), Id(Id) { - DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); - } - - RCMemoryManager(const RCMemoryManager &) = delete; - RCMemoryManager &operator=(const RCMemoryManager &) = delete; - RCMemoryManager(RCMemoryManager &&) = default; - RCMemoryManager &operator=(RCMemoryManager &&) = default; + /// Remote-mapped RuntimeDyld-compatible memory manager. + class RemoteRTDyldMemoryManager : public RuntimeDyld::MemoryManager { + friend class OrcRemoteTargetClient; - ~RCMemoryManager() override { + public: + ~RemoteRTDyldMemoryManager() { Client.destroyRemoteAllocator(Id); DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); } + RemoteRTDyldMemoryManager(const RemoteRTDyldMemoryManager &) = delete; + RemoteRTDyldMemoryManager & + operator=(const RemoteRTDyldMemoryManager &) = delete; + RemoteRTDyldMemoryManager(RemoteRTDyldMemoryManager &&) = default; + RemoteRTDyldMemoryManager & + operator=(RemoteRTDyldMemoryManager &&) = default; + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, StringRef SectionName) override { @@ -117,12 +116,8 @@ public: DEBUG(dbgs() << "Allocator " << Id << " reserved:\n"); if (CodeSize != 0) { - if (auto AddrOrErr = Client.reserveMem(Id, CodeSize, CodeAlign)) - Unmapped.back().RemoteCodeAddr = *AddrOrErr; - else { - // FIXME; Add error to poll. - assert(!AddrOrErr.takeError() && "Failed reserving remote memory."); - } + Unmapped.back().RemoteCodeAddr = + Client.reserveMem(Id, CodeSize, CodeAlign); DEBUG(dbgs() << " code: " << format("0x%016x", Unmapped.back().RemoteCodeAddr) @@ -131,12 +126,8 @@ public: } if (RODataSize != 0) { - if (auto AddrOrErr = Client.reserveMem(Id, RODataSize, RODataAlign)) - Unmapped.back().RemoteRODataAddr = *AddrOrErr; - else { - // FIXME; Add error to poll. - assert(!AddrOrErr.takeError() && "Failed reserving remote memory."); - } + Unmapped.back().RemoteRODataAddr = + Client.reserveMem(Id, RODataSize, RODataAlign); DEBUG(dbgs() << " ro-data: " << format("0x%016x", Unmapped.back().RemoteRODataAddr) @@ -145,12 +136,8 @@ public: } if (RWDataSize != 0) { - if (auto AddrOrErr = Client.reserveMem(Id, RWDataSize, RWDataAlign)) - Unmapped.back().RemoteRWDataAddr = *AddrOrErr; - else { - // FIXME; Add error to poll. - assert(!AddrOrErr.takeError() && "Failed reserving remote memory."); - } + Unmapped.back().RemoteRWDataAddr = + Client.reserveMem(Id, RWDataSize, RWDataAlign); DEBUG(dbgs() << " rw-data: " << format("0x%016x", Unmapped.back().RemoteRWDataAddr) @@ -168,10 +155,8 @@ public: void deregisterEHFrames() override { for (auto &Frame : RegisteredEHFrames) { - auto Err = Client.deregisterEHFrames(Frame.Addr, Frame.Size); // FIXME: Add error poll. - assert(!Err && "Failed to register remote EH frames."); - (void)Err; + Client.deregisterEHFrames(Frame.Addr, Frame.Size); } } @@ -179,44 +164,12 @@ public: const object::ObjectFile &Obj) override { DEBUG(dbgs() << "Allocator " << Id << " applied mappings:\n"); for (auto &ObjAllocs : Unmapped) { - { - JITTargetAddress NextCodeAddr = ObjAllocs.RemoteCodeAddr; - for (auto &Alloc : ObjAllocs.CodeAllocs) { - NextCodeAddr = alignTo(NextCodeAddr, Alloc.getAlign()); - Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextCodeAddr); - DEBUG(dbgs() << " code: " - << static_cast<void *>(Alloc.getLocalAddress()) - << " -> " << format("0x%016x", NextCodeAddr) << "\n"); - Alloc.setRemoteAddress(NextCodeAddr); - NextCodeAddr += Alloc.getSize(); - } - } - { - JITTargetAddress NextRODataAddr = ObjAllocs.RemoteRODataAddr; - for (auto &Alloc : ObjAllocs.RODataAllocs) { - NextRODataAddr = alignTo(NextRODataAddr, Alloc.getAlign()); - Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextRODataAddr); - DEBUG(dbgs() << " ro-data: " - << static_cast<void *>(Alloc.getLocalAddress()) - << " -> " << format("0x%016x", NextRODataAddr) - << "\n"); - Alloc.setRemoteAddress(NextRODataAddr); - NextRODataAddr += Alloc.getSize(); - } - } - { - JITTargetAddress NextRWDataAddr = ObjAllocs.RemoteRWDataAddr; - for (auto &Alloc : ObjAllocs.RWDataAllocs) { - NextRWDataAddr = alignTo(NextRWDataAddr, Alloc.getAlign()); - Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextRWDataAddr); - DEBUG(dbgs() << " rw-data: " - << static_cast<void *>(Alloc.getLocalAddress()) - << " -> " << format("0x%016x", NextRWDataAddr) - << "\n"); - Alloc.setRemoteAddress(NextRWDataAddr); - NextRWDataAddr += Alloc.getSize(); - } - } + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, + ObjAllocs.RemoteCodeAddr); + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs, + ObjAllocs.RemoteRODataAddr); + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs, + ObjAllocs.RemoteRWDataAddr); Unfinalized.push_back(std::move(ObjAllocs)); } Unmapped.clear(); @@ -226,114 +179,17 @@ public: DEBUG(dbgs() << "Allocator " << Id << " finalizing:\n"); for (auto &ObjAllocs : Unfinalized) { - for (auto &Alloc : ObjAllocs.CodeAllocs) { - DEBUG(dbgs() << " copying code: " - << static_cast<void *>(Alloc.getLocalAddress()) << " -> " - << format("0x%016x", Alloc.getRemoteAddress()) << " (" - << Alloc.getSize() << " bytes)\n"); - if (auto Err = - Client.writeMem(Alloc.getRemoteAddress(), - Alloc.getLocalAddress(), Alloc.getSize())) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return true; - } - } - - if (ObjAllocs.RemoteCodeAddr) { - DEBUG(dbgs() << " setting R-X permissions on code block: " - << format("0x%016x", ObjAllocs.RemoteCodeAddr) << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteCodeAddr, - sys::Memory::MF_READ | - sys::Memory::MF_EXEC)) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return true; - } - } + if (copyAndProtect(ObjAllocs.CodeAllocs, ObjAllocs.RemoteCodeAddr, + sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return true; - for (auto &Alloc : ObjAllocs.RODataAllocs) { - DEBUG(dbgs() << " copying ro-data: " - << static_cast<void *>(Alloc.getLocalAddress()) << " -> " - << format("0x%016x", Alloc.getRemoteAddress()) << " (" - << Alloc.getSize() << " bytes)\n"); - if (auto Err = - Client.writeMem(Alloc.getRemoteAddress(), - Alloc.getLocalAddress(), Alloc.getSize())) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return true; - } - } - - if (ObjAllocs.RemoteRODataAddr) { - DEBUG(dbgs() << " setting R-- permissions on ro-data block: " - << format("0x%016x", ObjAllocs.RemoteRODataAddr) - << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteRODataAddr, - sys::Memory::MF_READ)) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return false; - } - } - - for (auto &Alloc : ObjAllocs.RWDataAllocs) { - DEBUG(dbgs() << " copying rw-data: " - << static_cast<void *>(Alloc.getLocalAddress()) << " -> " - << format("0x%016x", Alloc.getRemoteAddress()) << " (" - << Alloc.getSize() << " bytes)\n"); - if (auto Err = - Client.writeMem(Alloc.getRemoteAddress(), - Alloc.getLocalAddress(), Alloc.getSize())) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return false; - } - } + if (copyAndProtect(ObjAllocs.RODataAllocs, ObjAllocs.RemoteRODataAddr, + sys::Memory::MF_READ)) + return true; - if (ObjAllocs.RemoteRWDataAddr) { - DEBUG(dbgs() << " setting RW- permissions on rw-data block: " - << format("0x%016x", ObjAllocs.RemoteRWDataAddr) - << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteRWDataAddr, - sys::Memory::MF_READ | - sys::Memory::MF_WRITE)) { - // FIXME: Replace this once finalizeMemory can return an Error. - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - if (ErrMsg) { - raw_string_ostream ErrOut(*ErrMsg); - EIB.log(ErrOut); - } - }); - return false; - } - } + if (copyAndProtect(ObjAllocs.RWDataAllocs, ObjAllocs.RemoteRWDataAddr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE)) + return true; } Unfinalized.clear(); @@ -402,6 +258,60 @@ public: std::vector<Alloc> CodeAllocs, RODataAllocs, RWDataAllocs; }; + RemoteRTDyldMemoryManager(OrcRemoteTargetClient &Client, + ResourceIdMgr::ResourceId Id) + : Client(Client), Id(Id) { + DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); + } + + // Maps all allocations in Allocs to aligned blocks + void mapAllocsToRemoteAddrs(RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, + JITTargetAddress NextAddr) { + for (auto &Alloc : Allocs) { + NextAddr = alignTo(NextAddr, Alloc.getAlign()); + Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextAddr); + DEBUG(dbgs() << " " << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " << format("0x%016x", NextAddr) << "\n"); + Alloc.setRemoteAddress(NextAddr); + + // Only advance NextAddr if it was non-null to begin with, + // otherwise leave it as null. + if (NextAddr) + NextAddr += Alloc.getSize(); + } + } + + // Copies data for each alloc in the list, then set permissions on the + // segment. + bool copyAndProtect(const std::vector<Alloc> &Allocs, + JITTargetAddress RemoteSegmentAddr, + unsigned Permissions) { + if (RemoteSegmentAddr) { + assert(!Allocs.empty() && "No sections in allocated segment"); + + for (auto &Alloc : Allocs) { + DEBUG(dbgs() << " copying section: " + << static_cast<void *>(Alloc.getLocalAddress()) << " -> " + << format("0x%016x", Alloc.getRemoteAddress()) << " (" + << Alloc.getSize() << " bytes)\n";); + + if (Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), + Alloc.getSize())) + return true; + } + + DEBUG(dbgs() << " setting " + << (Permissions & sys::Memory::MF_READ ? 'R' : '-') + << (Permissions & sys::Memory::MF_WRITE ? 'W' : '-') + << (Permissions & sys::Memory::MF_EXEC ? 'X' : '-') + << " permissions on block: " + << format("0x%016x", RemoteSegmentAddr) << "\n"); + if (Client.setProtections(Id, RemoteSegmentAddr, Permissions)) + return true; + } + return false; + } + OrcRemoteTargetClient &Client; ResourceIdMgr::ResourceId Id; std::vector<ObjectAllocs> Unmapped; @@ -416,17 +326,14 @@ public: }; /// Remote indirect stubs manager. - class RCIndirectStubsManager : public IndirectStubsManager { + class RemoteIndirectStubsManager : public IndirectStubsManager { public: - RCIndirectStubsManager(OrcRemoteTargetClient &Remote, - ResourceIdMgr::ResourceId Id) - : Remote(Remote), Id(Id) {} - - ~RCIndirectStubsManager() override { - if (auto Err = Remote.destroyIndirectStubsManager(Id)) { - // FIXME: Thread this error back to clients. - consumeError(std::move(Err)); - } + RemoteIndirectStubsManager(OrcRemoteTargetClient &Client, + ResourceIdMgr::ResourceId Id) + : Client(Client), Id(Id) {} + + ~RemoteIndirectStubsManager() override { + Client.destroyIndirectStubsManager(Id); } Error createStub(StringRef StubName, JITTargetAddress StubAddr, @@ -474,7 +381,7 @@ public: auto I = StubIndexes.find(Name); assert(I != StubIndexes.end() && "No stub pointer for symbol"); auto Key = I->second.first; - return Remote.writePointer(getPtrAddr(Key), NewAddr); + return Client.writePointer(getPtrAddr(Key), NewAddr); } private: @@ -484,12 +391,7 @@ public: unsigned NumStubs; }; - OrcRemoteTargetClient &Remote; - ResourceIdMgr::ResourceId Id; - std::vector<RemoteIndirectStubsInfo> RemoteIndirectStubsInfos; using StubKey = std::pair<uint16_t, uint16_t>; - std::vector<StubKey> FreeStubs; - StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; Error reserveStubs(unsigned NumStubs) { if (NumStubs <= FreeStubs.size()) @@ -500,7 +402,7 @@ public: JITTargetAddress PtrBase; unsigned NumStubsEmitted; - if (auto StubInfoOrErr = Remote.emitIndirectStubs(Id, NewStubsRequired)) + if (auto StubInfoOrErr = Client.emitIndirectStubs(Id, NewStubsRequired)) std::tie(StubBase, PtrBase, NumStubsEmitted) = *StubInfoOrErr; else return StubInfoOrErr.takeError(); @@ -519,58 +421,64 @@ public: auto Key = FreeStubs.back(); FreeStubs.pop_back(); StubIndexes[StubName] = std::make_pair(Key, StubFlags); - return Remote.writePointer(getPtrAddr(Key), InitAddr); + return Client.writePointer(getPtrAddr(Key), InitAddr); } JITTargetAddress getStubAddr(StubKey K) { assert(RemoteIndirectStubsInfos[K.first].StubBase != 0 && "Missing stub address"); return RemoteIndirectStubsInfos[K.first].StubBase + - K.second * Remote.getIndirectStubSize(); + K.second * Client.getIndirectStubSize(); } JITTargetAddress getPtrAddr(StubKey K) { assert(RemoteIndirectStubsInfos[K.first].PtrBase != 0 && "Missing pointer address"); return RemoteIndirectStubsInfos[K.first].PtrBase + - K.second * Remote.getPointerSize(); + K.second * Client.getPointerSize(); } + + OrcRemoteTargetClient &Client; + ResourceIdMgr::ResourceId Id; + std::vector<RemoteIndirectStubsInfo> RemoteIndirectStubsInfos; + std::vector<StubKey> FreeStubs; + StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; }; /// Remote compile callback manager. - class RCCompileCallbackManager : public JITCompileCallbackManager { + class RemoteCompileCallbackManager : public JITCompileCallbackManager { public: - RCCompileCallbackManager(JITTargetAddress ErrorHandlerAddress, - OrcRemoteTargetClient &Remote) - : JITCompileCallbackManager(ErrorHandlerAddress), Remote(Remote) {} + RemoteCompileCallbackManager(OrcRemoteTargetClient &Client, + JITTargetAddress ErrorHandlerAddress) + : JITCompileCallbackManager(ErrorHandlerAddress), Client(Client) {} private: - void grow() override { + Error grow() override { JITTargetAddress BlockAddr = 0; uint32_t NumTrampolines = 0; - if (auto TrampolineInfoOrErr = Remote.emitTrampolineBlock()) + if (auto TrampolineInfoOrErr = Client.emitTrampolineBlock()) std::tie(BlockAddr, NumTrampolines) = *TrampolineInfoOrErr; - else { - // FIXME: Return error. - llvm_unreachable("Failed to create trampolines"); - } + else + return TrampolineInfoOrErr.takeError(); - uint32_t TrampolineSize = Remote.getTrampolineSize(); + uint32_t TrampolineSize = Client.getTrampolineSize(); for (unsigned I = 0; I < NumTrampolines; ++I) this->AvailableTrampolines.push_back(BlockAddr + (I * TrampolineSize)); + + return Error::success(); } - OrcRemoteTargetClient &Remote; + OrcRemoteTargetClient &Client; }; /// Create an OrcRemoteTargetClient. /// Channel is the ChannelT instance to communicate on. It is assumed that /// the channel is ready to be read from and written to. static Expected<std::unique_ptr<OrcRemoteTargetClient>> - Create(ChannelT &Channel) { + Create(rpc::RawByteChannel &Channel, std::function<void(Error)> ReportError) { Error Err = Error::success(); - std::unique_ptr<OrcRemoteTargetClient> Client( - new OrcRemoteTargetClient(Channel, Err)); + auto Client = std::unique_ptr<OrcRemoteTargetClient>( + new OrcRemoteTargetClient(Channel, std::move(ReportError), Err)); if (Err) return std::move(Err); return std::move(Client); @@ -580,7 +488,7 @@ public: /// its result. Expected<int> callIntVoid(JITTargetAddress Addr) { DEBUG(dbgs() << "Calling int(*)(void) " << format("0x%016x", Addr) << "\n"); - return callB<CallIntVoid>(Addr); + return callB<exec::CallIntVoid>(Addr); } /// Call the int(int, char*[]) function at the given address in the target and @@ -589,7 +497,7 @@ public: const std::vector<std::string> &Args) { DEBUG(dbgs() << "Calling int(*)(int, char*[]) " << format("0x%016x", Addr) << "\n"); - return callB<CallMain>(Addr, Args); + return callB<exec::CallMain>(Addr, Args); } /// Call the void() function at the given address in the target and wait for @@ -597,45 +505,39 @@ public: Error callVoidVoid(JITTargetAddress Addr) { DEBUG(dbgs() << "Calling void(*)(void) " << format("0x%016x", Addr) << "\n"); - return callB<CallVoidVoid>(Addr); + return callB<exec::CallVoidVoid>(Addr); } /// Create an RCMemoryManager which will allocate its memory on the remote /// target. - Error createRemoteMemoryManager(std::unique_ptr<RCMemoryManager> &MM) { - assert(!MM && "MemoryManager should be null before creation."); - + Expected<std::unique_ptr<RemoteRTDyldMemoryManager>> + createRemoteMemoryManager() { auto Id = AllocatorIds.getNext(); - if (auto Err = callB<CreateRemoteAllocator>(Id)) - return Err; - MM = llvm::make_unique<RCMemoryManager>(*this, Id); - return Error::success(); + if (auto Err = callB<mem::CreateRemoteAllocator>(Id)) + return std::move(Err); + return std::unique_ptr<RemoteRTDyldMemoryManager>( + new RemoteRTDyldMemoryManager(*this, Id)); } /// Create an RCIndirectStubsManager that will allocate stubs on the remote /// target. - Error createIndirectStubsManager(std::unique_ptr<RCIndirectStubsManager> &I) { - assert(!I && "Indirect stubs manager should be null before creation."); + Expected<std::unique_ptr<RemoteIndirectStubsManager>> + createIndirectStubsManager() { auto Id = IndirectStubOwnerIds.getNext(); - if (auto Err = callB<CreateIndirectStubsOwner>(Id)) - return Err; - I = llvm::make_unique<RCIndirectStubsManager>(*this, Id); - return Error::success(); + if (auto Err = callB<stubs::CreateIndirectStubsOwner>(Id)) + return std::move(Err); + return llvm::make_unique<RemoteIndirectStubsManager>(*this, Id); } - Expected<RCCompileCallbackManager &> + Expected<RemoteCompileCallbackManager &> enableCompileCallbacks(JITTargetAddress ErrorHandlerAddress) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - // Emit the resolver block on the JIT server. - if (auto Err = callB<EmitResolverBlock>()) + if (auto Err = callB<stubs::EmitResolverBlock>()) return std::move(Err); // Create the callback manager. - CallbackManager.emplace(ErrorHandlerAddress, *this); - RCCompileCallbackManager &Mgr = *CallbackManager; + CallbackManager.emplace(*this, ErrorHandlerAddress); + RemoteCompileCallbackManager &Mgr = *CallbackManager; return Mgr; } @@ -643,45 +545,43 @@ public: /// symbol resolvers *after* they've searched the local symbol table in the /// JIT stack. Expected<JITTargetAddress> getSymbolAddress(StringRef Name) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<GetSymbolAddress>(Name); + return callB<utils::GetSymbolAddress>(Name); } /// Get the triple for the remote target. const std::string &getTargetTriple() const { return RemoteTargetTriple; } - Error terminateSession() { return callB<TerminateSession>(); } + Error terminateSession() { return callB<utils::TerminateSession>(); } private: - OrcRemoteTargetClient(ChannelT &Channel, Error &Err) - : OrcRemoteTargetRPCAPI(Channel) { + OrcRemoteTargetClient(rpc::RawByteChannel &Channel, + std::function<void(Error)> ReportError, Error &Err) + : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(Channel, true), + ReportError(std::move(ReportError)) { ErrorAsOutParameter EAO(&Err); - addHandler<RequestCompile>( + addHandler<utils::RequestCompile>( [this](JITTargetAddress Addr) -> JITTargetAddress { if (CallbackManager) return CallbackManager->executeCompileCallback(Addr); return 0; }); - if (auto RIOrErr = callB<GetRemoteInfo>()) { + if (auto RIOrErr = callB<utils::GetRemoteInfo>()) { std::tie(RemoteTargetTriple, RemotePointerSize, RemotePageSize, RemoteTrampolineSize, RemoteIndirectStubSize) = *RIOrErr; Err = Error::success(); - } else { - Err = joinErrors(RIOrErr.takeError(), std::move(ExistingError)); - } + } else + Err = RIOrErr.takeError(); } - Error deregisterEHFrames(JITTargetAddress Addr, uint32_t Size) { - return callB<RegisterEHFrames>(Addr, Size); + void deregisterEHFrames(JITTargetAddress Addr, uint32_t Size) { + if (auto Err = callB<eh::RegisterEHFrames>(Addr, Size)) + ReportError(std::move(Err)); } void destroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { - if (auto Err = callB<DestroyRemoteAllocator>(Id)) { + if (auto Err = callB<mem::DestroyRemoteAllocator>(Id)) { // FIXME: This will be triggered by a removeModuleSet call: Propagate // error return up through that. llvm_unreachable("Failed to destroy remote allocator."); @@ -689,22 +589,19 @@ private: } } - Error destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { + void destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { IndirectStubOwnerIds.release(Id); - return callB<DestroyIndirectStubsOwner>(Id); + if (auto Err = callB<stubs::DestroyIndirectStubsOwner>(Id)) + ReportError(std::move(Err)); } Expected<std::tuple<JITTargetAddress, JITTargetAddress, uint32_t>> emitIndirectStubs(ResourceIdMgr::ResourceId Id, uint32_t NumStubsRequired) { - return callB<EmitIndirectStubs>(Id, NumStubsRequired); + return callB<stubs::EmitIndirectStubs>(Id, NumStubsRequired); } Expected<std::tuple<JITTargetAddress, uint32_t>> emitTrampolineBlock() { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<EmitTrampolineBlock>(); + return callB<stubs::EmitTrampolineBlock>(); } uint32_t getIndirectStubSize() const { return RemoteIndirectStubSize; } @@ -713,59 +610,57 @@ private: uint32_t getTrampolineSize() const { return RemoteTrampolineSize; } - Expected<std::vector<char>> readMem(char *Dst, JITTargetAddress Src, - uint64_t Size) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<ReadMem>(Src, Size); + Expected<std::vector<uint8_t>> readMem(char *Dst, JITTargetAddress Src, + uint64_t Size) { + return callB<mem::ReadMem>(Src, Size); } Error registerEHFrames(JITTargetAddress &RAddr, uint32_t Size) { - return callB<RegisterEHFrames>(RAddr, Size); + // FIXME: Duplicate error and report it via ReportError too? + return callB<eh::RegisterEHFrames>(RAddr, Size); } - Expected<JITTargetAddress> reserveMem(ResourceIdMgr::ResourceId Id, - uint64_t Size, uint32_t Align) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<ReserveMem>(Id, Size, Align); + JITTargetAddress reserveMem(ResourceIdMgr::ResourceId Id, uint64_t Size, + uint32_t Align) { + if (auto AddrOrErr = callB<mem::ReserveMem>(Id, Size, Align)) + return *AddrOrErr; + else { + ReportError(AddrOrErr.takeError()); + return 0; + } } - Error setProtections(ResourceIdMgr::ResourceId Id, - JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { - return callB<SetProtections>(Id, RemoteSegAddr, ProtFlags); + bool setProtections(ResourceIdMgr::ResourceId Id, + JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { + if (auto Err = callB<mem::SetProtections>(Id, RemoteSegAddr, ProtFlags)) { + ReportError(std::move(Err)); + return true; + } else + return false; } - Error writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<WriteMem>(DirectBufferWriter(Src, Addr, Size)); + bool writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { + if (auto Err = callB<mem::WriteMem>(DirectBufferWriter(Src, Addr, Size))) { + ReportError(std::move(Err)); + return true; + } else + return false; } Error writePointer(JITTargetAddress Addr, JITTargetAddress PtrVal) { - // Check for an 'out-of-band' error, e.g. from an MM destructor. - if (ExistingError) - return std::move(ExistingError); - - return callB<WritePtr>(Addr, PtrVal); + return callB<mem::WritePtr>(Addr, PtrVal); } static Error doNothing() { return Error::success(); } - Error ExistingError = Error::success(); + std::function<void(Error)> ReportError; std::string RemoteTargetTriple; uint32_t RemotePointerSize = 0; uint32_t RemotePageSize = 0; uint32_t RemoteTrampolineSize = 0; uint32_t RemoteIndirectStubSize = 0; ResourceIdMgr AllocatorIds, IndirectStubOwnerIds; - Optional<RCCompileCallbackManager> CallbackManager; + Optional<RemoteCompileCallbackManager> CallbackManager; }; } // end namespace remote diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h index 07ae7f04d1a0..bc0da0f9a730 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -25,6 +25,37 @@ namespace orc { namespace remote { +/// Template error for missing resources. +template <typename ResourceIdT> +class ResourceNotFound + : public ErrorInfo<ResourceNotFound<ResourceIdT>> { +public: + static char ID; + + ResourceNotFound(ResourceIdT ResourceId, + std::string ResourceDescription = "") + : ResourceId(std::move(ResourceId)), + ResourceDescription(std::move(ResourceDescription)) {} + + std::error_code convertToErrorCode() const override { + return orcError(OrcErrorCode::UnknownResourceHandle); + } + + void log(raw_ostream &OS) const override { + OS << (ResourceDescription.empty() + ? "Remote resource with id " + : ResourceDescription) + << " " << ResourceId << " not found"; + } + +private: + ResourceIdT ResourceId; + std::string ResourceDescription; +}; + +template <typename ResourceIdT> +char ResourceNotFound<ResourceIdT>::ID = 0; + class DirectBufferWriter { public: DirectBufferWriter() = default; @@ -45,6 +76,32 @@ private: namespace rpc { +template <> +class RPCTypeName<JITSymbolFlags> { +public: + static const char *getName() { return "JITSymbolFlags"; } +}; + +template <typename ChannelT> +class SerializationTraits<ChannelT, JITSymbolFlags> { +public: + + static Error serialize(ChannelT &C, const JITSymbolFlags &Flags) { + return serializeSeq(C, static_cast<JITSymbolFlags::UnderlyingType>(Flags), + Flags.getTargetFlags()); + } + + static Error deserialize(ChannelT &C, JITSymbolFlags &Flags) { + JITSymbolFlags::UnderlyingType JITFlags; + JITSymbolFlags::TargetFlagsType TargetFlags; + if (auto Err = deserializeSeq(C, JITFlags, TargetFlags)) + return Err; + Flags = JITSymbolFlags(static_cast<JITSymbolFlags::FlagNames>(JITFlags), + TargetFlags); + return Error::success(); + } +}; + template <> class RPCTypeName<remote::DirectBufferWriter> { public: static const char *getName() { return "DirectBufferWriter"; } @@ -83,41 +140,66 @@ public: namespace remote { -class OrcRemoteTargetRPCAPI - : public rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel> { -protected: - class ResourceIdMgr { - public: - using ResourceId = uint64_t; - static const ResourceId InvalidId = ~0U; - - ResourceId getNext() { - if (!FreeIds.empty()) { - ResourceId I = FreeIds.back(); - FreeIds.pop_back(); - return I; - } - return NextId++; +class ResourceIdMgr { +public: + using ResourceId = uint64_t; + static const ResourceId InvalidId = ~0U; + + ResourceIdMgr() = default; + explicit ResourceIdMgr(ResourceId FirstValidId) + : NextId(std::move(FirstValidId)) {} + + ResourceId getNext() { + if (!FreeIds.empty()) { + ResourceId I = FreeIds.back(); + FreeIds.pop_back(); + return I; } + assert(NextId + 1 != ~0ULL && "All ids allocated"); + return NextId++; + } + + void release(ResourceId I) { FreeIds.push_back(I); } + +private: + ResourceId NextId = 1; + std::vector<ResourceId> FreeIds; +}; - void release(ResourceId I) { FreeIds.push_back(I); } +/// Registers EH frames on the remote. +namespace eh { - private: - ResourceId NextId = 0; - std::vector<ResourceId> FreeIds; + /// Registers EH frames on the remote. + class RegisterEHFrames + : public rpc::Function<RegisterEHFrames, + void(JITTargetAddress Addr, uint32_t Size)> { + public: + static const char *getName() { return "RegisterEHFrames"; } }; -public: - // FIXME: Remove constructors once MSVC supports synthesizing move-ops. - OrcRemoteTargetRPCAPI(rpc::RawByteChannel &C) - : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(C, true) {} + /// Deregisters EH frames on the remote. + class DeregisterEHFrames + : public rpc::Function<DeregisterEHFrames, + void(JITTargetAddress Addr, uint32_t Size)> { + public: + static const char *getName() { return "DeregisterEHFrames"; } + }; + +} // end namespace eh + +/// RPC functions for executing remote code. +namespace exec { + /// Call an 'int32_t()'-type function on the remote, returns the called + /// function's return value. class CallIntVoid : public rpc::Function<CallIntVoid, int32_t(JITTargetAddress Addr)> { public: static const char *getName() { return "CallIntVoid"; } }; + /// Call an 'int32_t(int32_t, char**)'-type function on the remote, returns the + /// called function's return value. class CallMain : public rpc::Function<CallMain, int32_t(JITTargetAddress Addr, std::vector<std::string> Args)> { @@ -125,12 +207,20 @@ public: static const char *getName() { return "CallMain"; } }; + /// Calls a 'void()'-type function on the remote, returns when the called + /// function completes. class CallVoidVoid : public rpc::Function<CallVoidVoid, void(JITTargetAddress FnAddr)> { public: static const char *getName() { return "CallVoidVoid"; } }; +} // end namespace exec + +/// RPC functions for remote memory management / inspection / modification. +namespace mem { + + /// Creates a memory allocator on the remote. class CreateRemoteAllocator : public rpc::Function<CreateRemoteAllocator, void(ResourceIdMgr::ResourceId AllocatorID)> { @@ -138,27 +228,68 @@ public: static const char *getName() { return "CreateRemoteAllocator"; } }; - class CreateIndirectStubsOwner - : public rpc::Function<CreateIndirectStubsOwner, - void(ResourceIdMgr::ResourceId StubOwnerID)> { + /// Destroys a remote allocator, freeing any memory allocated by it. + class DestroyRemoteAllocator + : public rpc::Function<DestroyRemoteAllocator, + void(ResourceIdMgr::ResourceId AllocatorID)> { public: - static const char *getName() { return "CreateIndirectStubsOwner"; } + static const char *getName() { return "DestroyRemoteAllocator"; } }; - class DeregisterEHFrames - : public rpc::Function<DeregisterEHFrames, - void(JITTargetAddress Addr, uint32_t Size)> { + /// Read a remote memory block. + class ReadMem + : public rpc::Function<ReadMem, std::vector<uint8_t>(JITTargetAddress Src, + uint64_t Size)> { public: - static const char *getName() { return "DeregisterEHFrames"; } + static const char *getName() { return "ReadMem"; } }; - class DestroyRemoteAllocator - : public rpc::Function<DestroyRemoteAllocator, - void(ResourceIdMgr::ResourceId AllocatorID)> { + /// Reserve a block of memory on the remote via the given allocator. + class ReserveMem + : public rpc::Function<ReserveMem, + JITTargetAddress(ResourceIdMgr::ResourceId AllocID, + uint64_t Size, uint32_t Align)> { public: - static const char *getName() { return "DestroyRemoteAllocator"; } + static const char *getName() { return "ReserveMem"; } + }; + + /// Set the memory protection on a memory block. + class SetProtections + : public rpc::Function<SetProtections, + void(ResourceIdMgr::ResourceId AllocID, + JITTargetAddress Dst, uint32_t ProtFlags)> { + public: + static const char *getName() { return "SetProtections"; } + }; + + /// Write to a remote memory block. + class WriteMem + : public rpc::Function<WriteMem, void(remote::DirectBufferWriter DB)> { + public: + static const char *getName() { return "WriteMem"; } + }; + + /// Write to a remote pointer. + class WritePtr : public rpc::Function<WritePtr, void(JITTargetAddress Dst, + JITTargetAddress Val)> { + public: + static const char *getName() { return "WritePtr"; } + }; + +} // end namespace mem + +/// RPC functions for remote stub and trampoline management. +namespace stubs { + + /// Creates an indirect stub owner on the remote. + class CreateIndirectStubsOwner + : public rpc::Function<CreateIndirectStubsOwner, + void(ResourceIdMgr::ResourceId StubOwnerID)> { + public: + static const char *getName() { return "CreateIndirectStubsOwner"; } }; + /// RPC function for destroying an indirect stubs owner. class DestroyIndirectStubsOwner : public rpc::Function<DestroyIndirectStubsOwner, void(ResourceIdMgr::ResourceId StubsOwnerID)> { @@ -177,6 +308,7 @@ public: static const char *getName() { return "EmitIndirectStubs"; } }; + /// RPC function to emit the resolver block and return its address. class EmitResolverBlock : public rpc::Function<EmitResolverBlock, void()> { public: static const char *getName() { return "EmitResolverBlock"; } @@ -190,12 +322,10 @@ public: static const char *getName() { return "EmitTrampolineBlock"; } }; - class GetSymbolAddress - : public rpc::Function<GetSymbolAddress, - JITTargetAddress(std::string SymbolName)> { - public: - static const char *getName() { return "GetSymbolAddress"; } - }; +} // end namespace stubs + +/// Miscelaneous RPC functions for dealing with remotes. +namespace utils { /// GetRemoteInfo result is (Triple, PointerSize, PageSize, TrampolineSize, /// IndirectStubsSize). @@ -207,28 +337,15 @@ public: static const char *getName() { return "GetRemoteInfo"; } }; - class ReadMem - : public rpc::Function<ReadMem, std::vector<uint8_t>(JITTargetAddress Src, - uint64_t Size)> { - public: - static const char *getName() { return "ReadMem"; } - }; - - class RegisterEHFrames - : public rpc::Function<RegisterEHFrames, - void(JITTargetAddress Addr, uint32_t Size)> { - public: - static const char *getName() { return "RegisterEHFrames"; } - }; - - class ReserveMem - : public rpc::Function<ReserveMem, - JITTargetAddress(ResourceIdMgr::ResourceId AllocID, - uint64_t Size, uint32_t Align)> { + /// Get the address of a remote symbol. + class GetSymbolAddress + : public rpc::Function<GetSymbolAddress, + JITTargetAddress(std::string SymbolName)> { public: - static const char *getName() { return "ReserveMem"; } + static const char *getName() { return "GetSymbolAddress"; } }; + /// Request that the host execute a compile callback. class RequestCompile : public rpc::Function< RequestCompile, JITTargetAddress(JITTargetAddress TrampolineAddr)> { @@ -236,30 +353,20 @@ public: static const char *getName() { return "RequestCompile"; } }; - class SetProtections - : public rpc::Function<SetProtections, - void(ResourceIdMgr::ResourceId AllocID, - JITTargetAddress Dst, uint32_t ProtFlags)> { - public: - static const char *getName() { return "SetProtections"; } - }; - + /// Notify the remote and terminate the session. class TerminateSession : public rpc::Function<TerminateSession, void()> { public: static const char *getName() { return "TerminateSession"; } }; - class WriteMem - : public rpc::Function<WriteMem, void(remote::DirectBufferWriter DB)> { - public: - static const char *getName() { return "WriteMem"; } - }; +} // namespace utils - class WritePtr : public rpc::Function<WritePtr, void(JITTargetAddress Dst, - JITTargetAddress Val)> { - public: - static const char *getName() { return "WritePtr"; } - }; +class OrcRemoteTargetRPCAPI + : public rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel> { +public: + // FIXME: Remove constructors once MSVC supports synthesizing move-ops. + OrcRemoteTargetRPCAPI(rpc::RawByteChannel &C) + : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(C, true) {} }; } // end namespace remote diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h index e7b6d64931b6..cf419d33004c 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -45,7 +45,8 @@ namespace orc { namespace remote { template <typename ChannelT, typename TargetT> -class OrcRemoteTargetServer : public OrcRemoteTargetRPCAPI { +class OrcRemoteTargetServer + : public rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel> { public: using SymbolLookupFtor = std::function<JITTargetAddress(const std::string &Name)>; @@ -56,34 +57,38 @@ public: OrcRemoteTargetServer(ChannelT &Channel, SymbolLookupFtor SymbolLookup, EHFrameRegistrationFtor EHFramesRegister, EHFrameRegistrationFtor EHFramesDeregister) - : OrcRemoteTargetRPCAPI(Channel), SymbolLookup(std::move(SymbolLookup)), + : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(Channel, true), + SymbolLookup(std::move(SymbolLookup)), EHFramesRegister(std::move(EHFramesRegister)), EHFramesDeregister(std::move(EHFramesDeregister)) { using ThisT = typename std::remove_reference<decltype(*this)>::type; - addHandler<CallIntVoid>(*this, &ThisT::handleCallIntVoid); - addHandler<CallMain>(*this, &ThisT::handleCallMain); - addHandler<CallVoidVoid>(*this, &ThisT::handleCallVoidVoid); - addHandler<CreateRemoteAllocator>(*this, - &ThisT::handleCreateRemoteAllocator); - addHandler<CreateIndirectStubsOwner>( + addHandler<exec::CallIntVoid>(*this, &ThisT::handleCallIntVoid); + addHandler<exec::CallMain>(*this, &ThisT::handleCallMain); + addHandler<exec::CallVoidVoid>(*this, &ThisT::handleCallVoidVoid); + addHandler<mem::CreateRemoteAllocator>(*this, + &ThisT::handleCreateRemoteAllocator); + addHandler<mem::DestroyRemoteAllocator>( + *this, &ThisT::handleDestroyRemoteAllocator); + addHandler<mem::ReadMem>(*this, &ThisT::handleReadMem); + addHandler<mem::ReserveMem>(*this, &ThisT::handleReserveMem); + addHandler<mem::SetProtections>(*this, &ThisT::handleSetProtections); + addHandler<mem::WriteMem>(*this, &ThisT::handleWriteMem); + addHandler<mem::WritePtr>(*this, &ThisT::handleWritePtr); + addHandler<eh::RegisterEHFrames>(*this, &ThisT::handleRegisterEHFrames); + addHandler<eh::DeregisterEHFrames>(*this, &ThisT::handleDeregisterEHFrames); + addHandler<stubs::CreateIndirectStubsOwner>( *this, &ThisT::handleCreateIndirectStubsOwner); - addHandler<DeregisterEHFrames>(*this, &ThisT::handleDeregisterEHFrames); - addHandler<DestroyRemoteAllocator>(*this, - &ThisT::handleDestroyRemoteAllocator); - addHandler<DestroyIndirectStubsOwner>( + addHandler<stubs::DestroyIndirectStubsOwner>( *this, &ThisT::handleDestroyIndirectStubsOwner); - addHandler<EmitIndirectStubs>(*this, &ThisT::handleEmitIndirectStubs); - addHandler<EmitResolverBlock>(*this, &ThisT::handleEmitResolverBlock); - addHandler<EmitTrampolineBlock>(*this, &ThisT::handleEmitTrampolineBlock); - addHandler<GetSymbolAddress>(*this, &ThisT::handleGetSymbolAddress); - addHandler<GetRemoteInfo>(*this, &ThisT::handleGetRemoteInfo); - addHandler<ReadMem>(*this, &ThisT::handleReadMem); - addHandler<RegisterEHFrames>(*this, &ThisT::handleRegisterEHFrames); - addHandler<ReserveMem>(*this, &ThisT::handleReserveMem); - addHandler<SetProtections>(*this, &ThisT::handleSetProtections); - addHandler<TerminateSession>(*this, &ThisT::handleTerminateSession); - addHandler<WriteMem>(*this, &ThisT::handleWriteMem); - addHandler<WritePtr>(*this, &ThisT::handleWritePtr); + addHandler<stubs::EmitIndirectStubs>(*this, + &ThisT::handleEmitIndirectStubs); + addHandler<stubs::EmitResolverBlock>(*this, + &ThisT::handleEmitResolverBlock); + addHandler<stubs::EmitTrampolineBlock>(*this, + &ThisT::handleEmitTrampolineBlock); + addHandler<utils::GetSymbolAddress>(*this, &ThisT::handleGetSymbolAddress); + addHandler<utils::GetRemoteInfo>(*this, &ThisT::handleGetRemoteInfo); + addHandler<utils::TerminateSession>(*this, &ThisT::handleTerminateSession); } // FIXME: Remove move/copy ops once MSVC supports synthesizing move ops. @@ -94,7 +99,7 @@ public: OrcRemoteTargetServer &operator=(OrcRemoteTargetServer &&) = delete; Expected<JITTargetAddress> requestCompile(JITTargetAddress TrampolineAddr) { - return callB<RequestCompile>(TrampolineAddr); + return callB<utils::RequestCompile>(TrampolineAddr); } bool receivedTerminate() const { return TerminateFlag; } diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h index 1cb2448a3a44..569c50602f3a 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h @@ -361,13 +361,13 @@ public: std::lock_guard<std::recursive_mutex> Lock(SerializersMutex); // FIXME: Move capture Serialize once we have C++14. Serializers[ErrorInfoT::classID()] = - [KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error { - assert(EIB.dynamicClassID() == ErrorInfoT::classID() && - "Serializer called for wrong error type"); - if (auto Err = serializeSeq(C, *KeyName)) - return Err; - return Serialize(C, static_cast<const ErrorInfoT&>(EIB)); - }; + [KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error { + assert(EIB.dynamicClassID() == ErrorInfoT::classID() && + "Serializer called for wrong error type"); + if (auto Err = serializeSeq(C, *KeyName)) + return Err; + return Serialize(C, static_cast<const ErrorInfoT &>(EIB)); + }; } } diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h index 6212f64ff319..c278cb176853 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -534,6 +534,20 @@ public: using ResultType = Error; }; +template <typename... ArgTs> +class AsyncHandlerTraits<ErrorSuccess(std::function<Error(Error)>, ArgTs...)> { +public: + using Type = Error(ArgTs...); + using ResultType = Error; +}; + +template <typename... ArgTs> +class AsyncHandlerTraits<void(std::function<Error(Error)>, ArgTs...)> { +public: + using Type = Error(ArgTs...); + using ResultType = Error; +}; + template <typename ResponseHandlerT, typename... ArgTs> class AsyncHandlerTraits<Error(ResponseHandlerT, ArgTs...)> : public AsyncHandlerTraits<Error(typename std::decay<ResponseHandlerT>::type, diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index e1016ef95f0c..246c57341f35 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -41,9 +41,9 @@ public: protected: - /// @brief Holds a set of objects to be allocated/linked as a unit in the JIT. + /// @brief Holds an object to be allocated/linked as a unit in the JIT. /// - /// An instance of this class will be created for each set of objects added + /// An instance of this class will be created for each object added /// via JITObjectLayer::addObject. Deleting the instance (via /// removeObject) frees its memory, removing all symbol definitions that /// had been provided by this instance. Higher level layers are responsible @@ -83,7 +83,7 @@ protected: using LinkedObjectListT = std::list<std::unique_ptr<LinkedObject>>; public: - /// @brief Handle to a set of loaded objects. + /// @brief Handle to a loaded object. using ObjHandleT = LinkedObjectListT::iterator; }; @@ -99,8 +99,9 @@ public: using RTDyldObjectLinkingLayerBase::ObjectPtr; /// @brief Functor for receiving object-loaded notifications. - using NotifyLoadedFtor = std::function<void(ObjHandleT, const ObjectPtr &Obj, - const LoadedObjectInfo &)>; + using NotifyLoadedFtor = + std::function<void(ObjHandleT, const ObjectPtr &Obj, + const RuntimeDyld::LoadedObjectInfo &)>; /// @brief Functor for receiving finalization notifications. using NotifyFinalizedFtor = std::function<void(ObjHandleT)>; @@ -253,10 +254,9 @@ public: this->ProcessAllSections = ProcessAllSections; } - /// @brief Add a set of objects (or archives) that will be treated as a unit - /// for the purposes of symbol lookup and memory management. + /// @brief Add an object to the JIT. /// - /// @return A handle that can be used to refer to the loaded objects (for + /// @return A handle that can be used to refer to the loaded object (for /// symbol searching, finalization, freeing memory, etc.). Expected<ObjHandleT> addObject(ObjectPtr Obj, std::shared_ptr<JITSymbolResolver> Resolver) { @@ -291,10 +291,10 @@ public: return Handle; } - /// @brief Remove the set of objects associated with handle H. + /// @brief Remove the object associated with handle H. /// - /// All memory allocated for the objects will be freed, and the sections and - /// symbols they provided will no longer be available. No attempt is made to + /// All memory allocated for the object will be freed, and the sections and + /// symbols it provided will no longer be available. No attempt is made to /// re-emit the missing symbols, and any use of these symbols (directly or /// indirectly) will result in undefined behavior. If dependence tracking is /// required to detect or resolve such issues it should be added at a higher @@ -318,27 +318,27 @@ public: return nullptr; } - /// @brief Search for the given named symbol in the context of the set of - /// loaded objects represented by the handle H. - /// @param H The handle for the object set to search in. + /// @brief Search for the given named symbol in the context of the loaded + /// object represented by the handle H. + /// @param H The handle for the object to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the - /// given object set. + /// given object. JITSymbol findSymbolIn(ObjHandleT H, StringRef Name, bool ExportedSymbolsOnly) { return (*H)->getSymbol(Name, ExportedSymbolsOnly); } - /// @brief Map section addresses for the objects associated with the handle H. + /// @brief Map section addresses for the object associated with the handle H. void mapSectionAddress(ObjHandleT H, const void *LocalAddress, JITTargetAddress TargetAddr) { (*H)->mapSectionAddress(LocalAddress, TargetAddr); } - /// @brief Immediately emit and finalize the object set represented by the - /// given handle. - /// @param H Handle for object set to emit/finalize. + /// @brief Immediately emit and finalize the object represented by the given + /// handle. + /// @param H Handle for object to emit/finalize. Error emitAndFinalize(ObjHandleT H) { (*H)->finalize(); return Error::success(); diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h new file mode 100644 index 000000000000..17255954a99f --- /dev/null +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -0,0 +1,538 @@ +//===------ RemoteObjectLayer.h - Forwards objs to a remote -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Forwards objects to a remote object layer via RPC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_REMOTEOBJECTLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_REMOTEOBJECTLAYER_H + +#include "llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include <map> + +namespace llvm { +namespace orc { + +/// RPC API needed by RemoteObjectClientLayer and RemoteObjectServerLayer. +class RemoteObjectLayerAPI { +public: + + using ObjHandleT = remote::ResourceIdMgr::ResourceId; + +protected: + + using RemoteSymbolId = remote::ResourceIdMgr::ResourceId; + using RemoteSymbol = std::pair<RemoteSymbolId, JITSymbolFlags>; + +public: + + using BadSymbolHandleError = remote::ResourceNotFound<RemoteSymbolId>; + using BadObjectHandleError = remote::ResourceNotFound<ObjHandleT>; + +protected: + + static const ObjHandleT InvalidObjectHandleId = 0; + static const RemoteSymbolId NullSymbolId = 0; + + class AddObject + : public rpc::Function<AddObject, Expected<ObjHandleT>(std::string)> { + public: + static const char *getName() { return "AddObject"; } + }; + + class RemoveObject + : public rpc::Function<RemoveObject, Error(ObjHandleT)> { + public: + static const char *getName() { return "RemoveObject"; } + }; + + class FindSymbol + : public rpc::Function<FindSymbol, Expected<RemoteSymbol>(std::string, + bool)> { + public: + static const char *getName() { return "FindSymbol"; } + }; + + class FindSymbolIn + : public rpc::Function<FindSymbolIn, + Expected<RemoteSymbol>(ObjHandleT, std::string, + bool)> { + public: + static const char *getName() { return "FindSymbolIn"; } + }; + + class EmitAndFinalize + : public rpc::Function<EmitAndFinalize, + Error(ObjHandleT)> { + public: + static const char *getName() { return "EmitAndFinalize"; } + }; + + class Lookup + : public rpc::Function<Lookup, + Expected<RemoteSymbol>(ObjHandleT, std::string)> { + public: + static const char *getName() { return "Lookup"; } + }; + + class LookupInLogicalDylib + : public rpc::Function<LookupInLogicalDylib, + Expected<RemoteSymbol>(ObjHandleT, std::string)> { + public: + static const char *getName() { return "LookupInLogicalDylib"; } + }; + + class ReleaseRemoteSymbol + : public rpc::Function<ReleaseRemoteSymbol, Error(RemoteSymbolId)> { + public: + static const char *getName() { return "ReleaseRemoteSymbol"; } + }; + + class MaterializeRemoteSymbol + : public rpc::Function<MaterializeRemoteSymbol, + Expected<JITTargetAddress>(RemoteSymbolId)> { + public: + static const char *getName() { return "MaterializeRemoteSymbol"; } + }; +}; + +/// Base class containing common utilities for RemoteObjectClientLayer and +/// RemoteObjectServerLayer. +template <typename RPCEndpoint> +class RemoteObjectLayer : public RemoteObjectLayerAPI { +public: + + RemoteObjectLayer(RPCEndpoint &Remote, + std::function<void(Error)> ReportError) + : Remote(Remote), ReportError(std::move(ReportError)), + SymbolIdMgr(NullSymbolId + 1) { + using ThisT = RemoteObjectLayer<RPCEndpoint>; + Remote.template addHandler<ReleaseRemoteSymbol>( + *this, &ThisT::handleReleaseRemoteSymbol); + Remote.template addHandler<MaterializeRemoteSymbol>( + *this, &ThisT::handleMaterializeRemoteSymbol); + } + +protected: + + /// This class is used as the symbol materializer for JITSymbols returned by + /// RemoteObjectLayerClient/RemoteObjectLayerServer -- the materializer knows + /// how to call back to the other RPC endpoint to get the address when + /// requested. + class RemoteSymbolMaterializer { + public: + + /// Construct a RemoteSymbolMaterializer for the given RemoteObjectLayer + /// with the given Id. + RemoteSymbolMaterializer(RemoteObjectLayer &C, + RemoteSymbolId Id) + : C(C), Id(Id) {} + + RemoteSymbolMaterializer(const RemoteSymbolMaterializer &Other) + : C(Other.C), Id(Other.Id) { + // FIXME: This is a horrible, auto_ptr-style, copy-as-move operation. + // It should be removed as soon as LLVM has C++14's generalized + // lambda capture (at which point the materializer can be moved + // into the lambda in remoteToJITSymbol below). + const_cast<RemoteSymbolMaterializer&>(Other).Id = 0; + } + + RemoteSymbolMaterializer& + operator=(const RemoteSymbolMaterializer&) = delete; + + /// Release the remote symbol. + ~RemoteSymbolMaterializer() { + if (Id) + C.releaseRemoteSymbol(Id); + } + + /// Materialize the symbol on the remote and get its address. + Expected<JITTargetAddress> materialize() { + auto Addr = C.materializeRemoteSymbol(Id); + Id = 0; + return Addr; + } + + private: + RemoteObjectLayer &C; + RemoteSymbolId Id; + }; + + /// Convenience function for getting a null remote symbol value. + RemoteSymbol nullRemoteSymbol() { + return RemoteSymbol(0, JITSymbolFlags()); + } + + /// Creates a StringError that contains a copy of Err's log message, then + /// sends that StringError to ReportError. + /// + /// This allows us to locally log error messages for errors that will actually + /// be delivered to the remote. + Error teeLog(Error Err) { + return handleErrors(std::move(Err), + [this](std::unique_ptr<ErrorInfoBase> EIB) { + ReportError(make_error<StringError>( + EIB->message(), + EIB->convertToErrorCode())); + return Error(std::move(EIB)); + }); + } + + Error badRemoteSymbolIdError(RemoteSymbolId Id) { + return make_error<BadSymbolHandleError>(Id, "Remote JIT Symbol"); + } + + Error badObjectHandleError(ObjHandleT H) { + return make_error<RemoteObjectLayerAPI::BadObjectHandleError>( + H, "Bad object handle"); + } + + /// Create a RemoteSymbol wrapping the given JITSymbol. + Expected<RemoteSymbol> jitSymbolToRemote(JITSymbol Sym) { + if (Sym) { + auto Id = SymbolIdMgr.getNext(); + auto Flags = Sym.getFlags(); + assert(!InUseSymbols.count(Id) && "Symbol id already in use"); + InUseSymbols.insert(std::make_pair(Id, std::move(Sym))); + return RemoteSymbol(Id, Flags); + } else if (auto Err = Sym.takeError()) + return teeLog(std::move(Err)); + // else... + return nullRemoteSymbol(); + } + + /// Convert an Expected<RemoteSymbol> to a JITSymbol. + JITSymbol remoteToJITSymbol(Expected<RemoteSymbol> RemoteSymOrErr) { + if (RemoteSymOrErr) { + auto &RemoteSym = *RemoteSymOrErr; + if (RemoteSym == nullRemoteSymbol()) + return nullptr; + // else... + RemoteSymbolMaterializer RSM(*this, RemoteSym.first); + auto Sym = + JITSymbol([RSM]() mutable { return RSM.materialize(); }, + RemoteSym.second); + return Sym; + } else + return RemoteSymOrErr.takeError(); + } + + RPCEndpoint &Remote; + std::function<void(Error)> ReportError; + +private: + + /// Notify the remote to release the given JITSymbol. + void releaseRemoteSymbol(RemoteSymbolId Id) { + if (auto Err = Remote.template callB<ReleaseRemoteSymbol>(Id)) + ReportError(std::move(Err)); + } + + /// Notify the remote to materialize the JITSymbol with the given Id and + /// return its address. + Expected<JITTargetAddress> materializeRemoteSymbol(RemoteSymbolId Id) { + return Remote.template callB<MaterializeRemoteSymbol>(Id); + } + + /// Release the JITSymbol with the given Id. + Error handleReleaseRemoteSymbol(RemoteSymbolId Id) { + auto SI = InUseSymbols.find(Id); + if (SI != InUseSymbols.end()) { + InUseSymbols.erase(SI); + return Error::success(); + } else + return teeLog(badRemoteSymbolIdError(Id)); + } + + /// Run the materializer for the JITSymbol with the given Id and return its + /// address. + Expected<JITTargetAddress> handleMaterializeRemoteSymbol(RemoteSymbolId Id) { + auto SI = InUseSymbols.find(Id); + if (SI != InUseSymbols.end()) { + auto AddrOrErr = SI->second.getAddress(); + InUseSymbols.erase(SI); + SymbolIdMgr.release(Id); + if (AddrOrErr) + return *AddrOrErr; + else + return teeLog(AddrOrErr.takeError()); + } else { + return teeLog(badRemoteSymbolIdError(Id)); + } + } + + remote::ResourceIdMgr SymbolIdMgr; + std::map<RemoteSymbolId, JITSymbol> InUseSymbols; +}; + +/// RemoteObjectClientLayer forwards the ORC Object Layer API over an RPC +/// connection. +/// +/// This class can be used as the base layer of a JIT stack on the client and +/// will forward operations to a corresponding RemoteObjectServerLayer on the +/// server (which can be composed on top of a "real" object layer like +/// RTDyldObjectLinkingLayer to actually carry out the operations). +/// +/// Sending relocatable objects to the server (rather than fully relocated +/// bits) allows JIT'd code to be cached on the server side and re-used in +/// subsequent JIT sessions. +template <typename RPCEndpoint> +class RemoteObjectClientLayer : public RemoteObjectLayer<RPCEndpoint> { +private: + + using AddObject = RemoteObjectLayerAPI::AddObject; + using RemoveObject = RemoteObjectLayerAPI::RemoveObject; + using FindSymbol = RemoteObjectLayerAPI::FindSymbol; + using FindSymbolIn = RemoteObjectLayerAPI::FindSymbolIn; + using EmitAndFinalize = RemoteObjectLayerAPI::EmitAndFinalize; + using Lookup = RemoteObjectLayerAPI::Lookup; + using LookupInLogicalDylib = RemoteObjectLayerAPI::LookupInLogicalDylib; + + using RemoteObjectLayer<RPCEndpoint>::teeLog; + using RemoteObjectLayer<RPCEndpoint>::badObjectHandleError; + using RemoteObjectLayer<RPCEndpoint>::remoteToJITSymbol; + +public: + + using ObjHandleT = RemoteObjectLayerAPI::ObjHandleT; + using RemoteSymbol = RemoteObjectLayerAPI::RemoteSymbol; + + using ObjectPtr = + std::shared_ptr<object::OwningBinary<object::ObjectFile>>; + + /// Create a RemoteObjectClientLayer that communicates with a + /// RemoteObjectServerLayer instance via the given RPCEndpoint. + /// + /// The ReportError functor can be used locally log errors that are intended + /// to be sent sent + RemoteObjectClientLayer(RPCEndpoint &Remote, + std::function<void(Error)> ReportError) + : RemoteObjectLayer<RPCEndpoint>(Remote, std::move(ReportError)) { + using ThisT = RemoteObjectClientLayer<RPCEndpoint>; + Remote.template addHandler<Lookup>(*this, &ThisT::lookup); + Remote.template addHandler<LookupInLogicalDylib>( + *this, &ThisT::lookupInLogicalDylib); + } + + /// @brief Add an object to the JIT. + /// + /// @return A handle that can be used to refer to the loaded object (for + /// symbol searching, finalization, freeing memory, etc.). + Expected<ObjHandleT> + addObject(ObjectPtr Object, std::shared_ptr<JITSymbolResolver> Resolver) { + StringRef ObjBuffer = Object->getBinary()->getData(); + if (auto HandleOrErr = + this->Remote.template callB<AddObject>(ObjBuffer)) { + auto &Handle = *HandleOrErr; + // FIXME: Return an error for this: + assert(!Resolvers.count(Handle) && "Handle already in use?"); + Resolvers[Handle] = std::move(Resolver); + return Handle; + } else + return HandleOrErr.takeError(); + } + + /// @brief Remove the given object from the JIT. + Error removeObject(ObjHandleT H) { + return this->Remote.template callB<RemoveObject>(H); + } + + /// @brief Search for the given named symbol. + JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { + return remoteToJITSymbol( + this->Remote.template callB<FindSymbol>(Name, + ExportedSymbolsOnly)); + } + + /// @brief Search for the given named symbol within the given context. + JITSymbol findSymbolIn(ObjHandleT H, StringRef Name, bool ExportedSymbolsOnly) { + return remoteToJITSymbol( + this->Remote.template callB<FindSymbolIn>(H, Name, + ExportedSymbolsOnly)); + } + + /// @brief Immediately emit and finalize the object with the given handle. + Error emitAndFinalize(ObjHandleT H) { + return this->Remote.template callB<EmitAndFinalize>(H); + } + +private: + + Expected<RemoteSymbol> lookup(ObjHandleT H, const std::string &Name) { + auto RI = Resolvers.find(H); + if (RI != Resolvers.end()) { + return this->jitSymbolToRemote(RI->second->findSymbol(Name)); + } else + return teeLog(badObjectHandleError(H)); + } + + Expected<RemoteSymbol> lookupInLogicalDylib(ObjHandleT H, + const std::string &Name) { + auto RI = Resolvers.find(H); + if (RI != Resolvers.end()) + return this->jitSymbolToRemote( + RI->second->findSymbolInLogicalDylib(Name)); + else + return teeLog(badObjectHandleError(H)); + } + + std::map<remote::ResourceIdMgr::ResourceId, + std::shared_ptr<JITSymbolResolver>> Resolvers; +}; + +/// RemoteObjectServerLayer acts as a server and handling RPC calls for the +/// object layer API from the given RPC connection. +/// +/// This class can be composed on top of a 'real' object layer (e.g. +/// RTDyldObjectLinkingLayer) to do the actual work of relocating objects +/// and making them executable. +template <typename BaseLayerT, typename RPCEndpoint> +class RemoteObjectServerLayer : public RemoteObjectLayer<RPCEndpoint> { +private: + + using ObjHandleT = RemoteObjectLayerAPI::ObjHandleT; + using RemoteSymbol = RemoteObjectLayerAPI::RemoteSymbol; + + using AddObject = RemoteObjectLayerAPI::AddObject; + using RemoveObject = RemoteObjectLayerAPI::RemoveObject; + using FindSymbol = RemoteObjectLayerAPI::FindSymbol; + using FindSymbolIn = RemoteObjectLayerAPI::FindSymbolIn; + using EmitAndFinalize = RemoteObjectLayerAPI::EmitAndFinalize; + using Lookup = RemoteObjectLayerAPI::Lookup; + using LookupInLogicalDylib = RemoteObjectLayerAPI::LookupInLogicalDylib; + + using RemoteObjectLayer<RPCEndpoint>::teeLog; + using RemoteObjectLayer<RPCEndpoint>::badObjectHandleError; + using RemoteObjectLayer<RPCEndpoint>::remoteToJITSymbol; + +public: + + /// Create a RemoteObjectServerLayer with the given base layer (which must be + /// an object layer), RPC endpoint, and error reporter function. + RemoteObjectServerLayer(BaseLayerT &BaseLayer, + RPCEndpoint &Remote, + std::function<void(Error)> ReportError) + : RemoteObjectLayer<RPCEndpoint>(Remote, std::move(ReportError)), + BaseLayer(BaseLayer), HandleIdMgr(1) { + using ThisT = RemoteObjectServerLayer<BaseLayerT, RPCEndpoint>; + + Remote.template addHandler<AddObject>(*this, &ThisT::addObject); + Remote.template addHandler<RemoveObject>(*this, &ThisT::removeObject); + Remote.template addHandler<FindSymbol>(*this, &ThisT::findSymbol); + Remote.template addHandler<FindSymbolIn>(*this, &ThisT::findSymbolIn); + Remote.template addHandler<EmitAndFinalize>(*this, &ThisT::emitAndFinalize); + } + +private: + + class StringMemoryBuffer : public MemoryBuffer { + public: + StringMemoryBuffer(std::string Buffer) + : Buffer(std::move(Buffer)) { + init(this->Buffer.data(), this->Buffer.data() + this->Buffer.size(), + false); + } + + BufferKind getBufferKind() const override { return MemoryBuffer_Malloc; } + private: + std::string Buffer; + }; + + JITSymbol lookup(ObjHandleT Id, const std::string &Name) { + return remoteToJITSymbol( + this->Remote.template callB<Lookup>(Id, Name)); + } + + JITSymbol lookupInLogicalDylib(ObjHandleT Id, const std::string &Name) { + return remoteToJITSymbol( + this->Remote.template callB<LookupInLogicalDylib>(Id, Name)); + } + + Expected<ObjHandleT> addObject(std::string ObjBuffer) { + auto Buffer = llvm::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); + if (auto ObjectOrErr = + object::ObjectFile::createObjectFile(Buffer->getMemBufferRef())) { + auto Object = + std::make_shared<object::OwningBinary<object::ObjectFile>>( + std::move(*ObjectOrErr), std::move(Buffer)); + + auto Id = HandleIdMgr.getNext(); + assert(!BaseLayerHandles.count(Id) && "Id already in use?"); + + auto Resolver = + createLambdaResolver( + [this, Id](const std::string &Name) { return lookup(Id, Name); }, + [this, Id](const std::string &Name) { + return lookupInLogicalDylib(Id, Name); + }); + + if (auto HandleOrErr = + BaseLayer.addObject(std::move(Object), std::move(Resolver))) { + BaseLayerHandles[Id] = std::move(*HandleOrErr); + return Id; + } else + return teeLog(HandleOrErr.takeError()); + } else + return teeLog(ObjectOrErr.takeError()); + } + + Error removeObject(ObjHandleT H) { + auto HI = BaseLayerHandles.find(H); + if (HI != BaseLayerHandles.end()) { + if (auto Err = BaseLayer.removeObject(HI->second)) + return teeLog(std::move(Err)); + return Error::success(); + } else + return teeLog(badObjectHandleError(H)); + } + + Expected<RemoteSymbol> findSymbol(const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = BaseLayer.findSymbol(Name, ExportedSymbolsOnly)) + return this->jitSymbolToRemote(std::move(Sym)); + else if (auto Err = Sym.takeError()) + return teeLog(std::move(Err)); + return this->nullRemoteSymbol(); + } + + Expected<RemoteSymbol> findSymbolIn(ObjHandleT H, const std::string &Name, + bool ExportedSymbolsOnly) { + auto HI = BaseLayerHandles.find(H); + if (HI != BaseLayerHandles.end()) { + if (auto Sym = BaseLayer.findSymbolIn(HI->second, Name, ExportedSymbolsOnly)) + return this->jitSymbolToRemote(std::move(Sym)); + else if (auto Err = Sym.takeError()) + return teeLog(std::move(Err)); + return this->nullRemoteSymbol(); + } else + return teeLog(badObjectHandleError(H)); + } + + Error emitAndFinalize(ObjHandleT H) { + auto HI = BaseLayerHandles.find(H); + if (HI != BaseLayerHandles.end()) { + if (auto Err = BaseLayer.emitAndFinalize(HI->second)) + return teeLog(std::move(Err)); + return Error::success(); + } else + return teeLog(badObjectHandleError(H)); + } + + BaseLayerT &BaseLayer; + remote::ResourceIdMgr HandleIdMgr; + std::map<ObjHandleT, typename BaseLayerT::ObjHandleT> BaseLayerHandles; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_REMOTEOBJECTLAYER_H diff --git a/contrib/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/contrib/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h new file mode 100644 index 000000000000..b01fbd44bacd --- /dev/null +++ b/contrib/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -0,0 +1,136 @@ +//===- SymbolStringPool.h - Multi-threaded pool for JIT symbols -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Contains a multi-threaded string pool suitable for use with ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H +#define LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H + +#include "llvm/ADT/StringMap.h" +#include <atomic> +#include <mutex> + +namespace llvm { +namespace orc { + +class SymbolStringPtr; + +/// @brief String pool for symbol names used by the JIT. +class SymbolStringPool { + friend class SymbolStringPtr; +public: + /// @brief Create a symbol string pointer from the given string. + SymbolStringPtr intern(StringRef S); + + /// @brief Remove from the pool any entries that are no longer referenced. + void clearDeadEntries(); + + /// @brief Returns true if the pool is empty. + bool empty() const; +private: + using RefCountType = std::atomic<uint64_t>; + using PoolMap = StringMap<RefCountType>; + using PoolMapEntry = StringMapEntry<RefCountType>; + mutable std::mutex PoolMutex; + PoolMap Pool; +}; + +/// @brief Pointer to a pooled string representing a symbol name. +class SymbolStringPtr { + friend class SymbolStringPool; +public: + SymbolStringPtr() = default; + SymbolStringPtr(const SymbolStringPtr &Other) + : S(Other.S) { + if (S) + ++S->getValue(); + } + + SymbolStringPtr& operator=(const SymbolStringPtr &Other) { + if (S) + --S->getValue(); + S = Other.S; + if (S) + ++S->getValue(); + return *this; + } + + SymbolStringPtr(SymbolStringPtr &&Other) : S(nullptr) { + std::swap(S, Other.S); + } + + SymbolStringPtr& operator=(SymbolStringPtr &&Other) { + if (S) + --S->getValue(); + S = nullptr; + std::swap(S, Other.S); + return *this; + } + + ~SymbolStringPtr() { + if (S) + --S->getValue(); + } + + bool operator==(const SymbolStringPtr &Other) const { + return S == Other.S; + } + + bool operator!=(const SymbolStringPtr &Other) const { + return !(*this == Other); + } + + bool operator<(const SymbolStringPtr &Other) const { + return S->getValue() < Other.S->getValue(); + } + +private: + + SymbolStringPtr(SymbolStringPool::PoolMapEntry *S) + : S(S) { + if (S) + ++S->getValue(); + } + + SymbolStringPool::PoolMapEntry *S = nullptr; +}; + +inline SymbolStringPtr SymbolStringPool::intern(StringRef S) { + std::lock_guard<std::mutex> Lock(PoolMutex); + auto I = Pool.find(S); + if (I != Pool.end()) + return SymbolStringPtr(&*I); + + bool Added; + std::tie(I, Added) = Pool.try_emplace(S, 0); + assert(Added && "Insert should always succeed here"); + return SymbolStringPtr(&*I); +} + +inline void SymbolStringPool::clearDeadEntries() { + std::lock_guard<std::mutex> Lock(PoolMutex); + for (auto I = Pool.begin(), E = Pool.end(); I != E;) { + auto Tmp = std::next(I); + if (I->second == 0) + Pool.erase(I); + I = Tmp; + } +} + +inline bool SymbolStringPool::empty() const { + std::lock_guard<std::mutex> Lock(PoolMutex); + return Pool.empty(); +} + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H diff --git a/contrib/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h b/contrib/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h index 3b2af11cdaf4..d76e37113c66 100644 --- a/contrib/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/contrib/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -40,9 +40,75 @@ namespace llvm { /// directly. Clients of MCJIT should call MCJIT::finalizeObject. class SectionMemoryManager : public RTDyldMemoryManager { public: - SectionMemoryManager() = default; - SectionMemoryManager(const SectionMemoryManager&) = delete; - void operator=(const SectionMemoryManager&) = delete; + /// This enum describes the various reasons to allocate pages from + /// allocateMappedMemory. + enum class AllocationPurpose { + Code, + ROData, + RWData, + }; + + /// Implementations of this interface are used by SectionMemoryManager to + /// request pages from the operating system. + class MemoryMapper { + public: + /// This method attempts to allocate \p NumBytes bytes of virtual memory for + /// \p Purpose. \p NearBlock may point to an existing allocation, in which + /// case an attempt is made to allocate more memory near the existing block. + /// The actual allocated address is not guaranteed to be near the requested + /// address. \p Flags is used to set the initial protection flags for the + /// block of the memory. \p EC [out] returns an object describing any error + /// that occurs. + /// + /// This method may allocate more than the number of bytes requested. The + /// actual number of bytes allocated is indicated in the returned + /// MemoryBlock. + /// + /// The start of the allocated block must be aligned with the system + /// allocation granularity (64K on Windows, page size on Linux). If the + /// address following \p NearBlock is not so aligned, it will be rounded up + /// to the next allocation granularity boundary. + /// + /// \r a non-null MemoryBlock if the function was successful, otherwise a + /// null MemoryBlock with \p EC describing the error. + virtual sys::MemoryBlock + allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) = 0; + + /// This method sets the protection flags for a block of memory to the state + /// specified by \p Flags. The behavior is not specified if the memory was + /// not allocated using the allocateMappedMemory method. + /// \p Block describes the memory block to be protected. + /// \p Flags specifies the new protection state to be assigned to the block. + /// + /// If \p Flags is MF_WRITE, the actual behavior varies with the operating + /// system (i.e. MF_READ | MF_WRITE on Windows) and the target architecture + /// (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) = 0; + + /// This method releases a block of memory that was allocated with the + /// allocateMappedMemory method. It should not be used to release any memory + /// block allocated any other way. + /// \p Block describes the memory to be released. + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; + + virtual ~MemoryMapper(); + }; + + /// Creates a SectionMemoryManager instance with \p MM as the associated + /// memory mapper. If \p MM is nullptr then a default memory mapper is used + /// that directly calls into the operating system. + SectionMemoryManager(MemoryMapper *MM = nullptr); + SectionMemoryManager(const SectionMemoryManager &) = delete; + void operator=(const SectionMemoryManager &) = delete; ~SectionMemoryManager() override; /// \brief Allocates a memory block of (at least) the given size suitable for @@ -110,7 +176,7 @@ private: sys::MemoryBlock Near; }; - uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size, + uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, unsigned Alignment); std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, @@ -119,6 +185,7 @@ private: MemoryGroup CodeMem; MemoryGroup RWDataMem; MemoryGroup RODataMem; + MemoryMapper &MMapper; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/FuzzMutate/FuzzerCLI.h b/contrib/llvm/include/llvm/FuzzMutate/FuzzerCLI.h new file mode 100644 index 000000000000..a775fdfb603e --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/FuzzerCLI.h @@ -0,0 +1,73 @@ +//===-- FuzzerCLI.h - Common logic for CLIs of fuzzers ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common logic needed to implement LLVM's fuzz targets' CLIs - including LLVM +// concepts like cl::opt and libFuzzer concepts like -ignore_remaining_args=1. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_FUZZER_CLI_H +#define LLVM_FUZZMUTATE_FUZZER_CLI_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Support/DataTypes.h" + +namespace llvm { + +/// Parse cl::opts from a fuzz target commandline. +/// +/// This handles all arguments after -ignore_remaining_args=1 as cl::opts. +void parseFuzzerCLOpts(int ArgC, char *ArgV[]); + +/// Handle backend options that are encoded in the executable name. +/// +/// Parses some common backend options out of a specially crafted executable +/// name (argv[0]). For example, a name like llvm-foo-fuzzer--aarch64-gisel +/// might set up an AArch64 triple and the Global ISel selector. This should be +/// called *before* parseFuzzerCLOpts if calling both. +/// +/// This is meant to be used for environments like OSS-Fuzz that aren't capable +/// of passing in command line arguments in the normal way. +void handleExecNameEncodedBEOpts(StringRef ExecName); + +/// Handle optimizer options which are encoded in the executable name. +/// Same semantics as in 'handleExecNameEncodedBEOpts'. +void handleExecNameEncodedOptimizerOpts(StringRef ExecName); + +using FuzzerTestFun = int (*)(const uint8_t *Data, size_t Size); +using FuzzerInitFun = int (*)(int *argc, char ***argv); + +/// Runs a fuzz target on the inputs specified on the command line. +/// +/// Useful for testing fuzz targets without linking to libFuzzer. Finds inputs +/// in the argument list in a libFuzzer compatible way. +int runFuzzerOnInputs(int ArgC, char *ArgV[], FuzzerTestFun TestOne, + FuzzerInitFun Init = [](int *, char ***) { return 0; }); + +/// Fuzzer friendly interface for the llvm bitcode parser. +/// +/// \param Data Bitcode we are going to parse +/// \param Size Size of the 'Data' in bytes +/// \return New module or nullptr in case of error +std::unique_ptr<Module> parseModule(const uint8_t *Data, size_t Size, + LLVMContext &Context); + +/// Fuzzer friendly interface for the llvm bitcode printer. +/// +/// \param M Module to print +/// \param Dest Location to store serialized module +/// \param MaxSize Size of the destination buffer +/// \return Number of bytes that were written. When module size exceeds MaxSize +/// returns 0 and leaves Dest unchanged. +size_t writeModule(const Module &M, uint8_t *Dest, size_t MaxSize); + +} // end llvm namespace + +#endif // LLVM_FUZZMUTATE_FUZZER_CLI_H diff --git a/contrib/llvm/include/llvm/FuzzMutate/IRMutator.h b/contrib/llvm/include/llvm/FuzzMutate/IRMutator.h new file mode 100644 index 000000000000..65ab871db0ef --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/IRMutator.h @@ -0,0 +1,106 @@ +//===-- IRMutator.h - Mutation engine for fuzzing IR ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Provides the IRMutator class, which drives mutations on IR based on a +// configurable set of strategies. Some common strategies are also included +// here. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_IRMUTATOR_H +#define LLVM_FUZZMUTATE_IRMUTATOR_H + +#include "llvm/FuzzMutate/OpDescriptor.h" +#include "llvm/Support/ErrorHandling.h" + +namespace llvm { +class BasicBlock; +class Function; +class Instruction; +class Module; + +struct RandomIRBuilder; + +/// Base class for describing how to mutate a module. mutation functions for +/// each IR unit forward to the contained unit. +class IRMutationStrategy { +public: + virtual ~IRMutationStrategy() = default; + + /// Provide a weight to bias towards choosing this strategy for a mutation. + /// + /// The value of the weight is arbitrary, but a good default is "the number of + /// distinct ways in which this strategy can mutate a unit". This can also be + /// used to prefer strategies that shrink the overall size of the result when + /// we start getting close to \c MaxSize. + virtual uint64_t getWeight(size_t CurrentSize, size_t MaxSize, + uint64_t CurrentWeight) = 0; + + /// @{ + /// Mutators for each IR unit. By default these forward to a contained + /// instance of the next smaller unit. + virtual void mutate(Module &M, RandomIRBuilder &IB); + virtual void mutate(Function &F, RandomIRBuilder &IB); + virtual void mutate(BasicBlock &BB, RandomIRBuilder &IB); + virtual void mutate(Instruction &I, RandomIRBuilder &IB) { + llvm_unreachable("Strategy does not implement any mutators"); + } + /// @} +}; + +using TypeGetter = std::function<Type *(LLVMContext &)>; + +/// Entry point for configuring and running IR mutations. +class IRMutator { + std::vector<TypeGetter> AllowedTypes; + std::vector<std::unique_ptr<IRMutationStrategy>> Strategies; + +public: + IRMutator(std::vector<TypeGetter> &&AllowedTypes, + std::vector<std::unique_ptr<IRMutationStrategy>> &&Strategies) + : AllowedTypes(std::move(AllowedTypes)), + Strategies(std::move(Strategies)) {} + + void mutateModule(Module &M, int Seed, size_t CurSize, size_t MaxSize); +}; + +/// Strategy that injects operations into the function. +class InjectorIRStrategy : public IRMutationStrategy { + std::vector<fuzzerop::OpDescriptor> Operations; + + fuzzerop::OpDescriptor chooseOperation(Value *Src, RandomIRBuilder &IB); + +public: + InjectorIRStrategy(std::vector<fuzzerop::OpDescriptor> &&Operations) + : Operations(std::move(Operations)) {} + static std::vector<fuzzerop::OpDescriptor> getDefaultOps(); + + uint64_t getWeight(size_t CurrentSize, size_t MaxSize, + uint64_t CurrentWeight) override { + return Operations.size(); + } + + using IRMutationStrategy::mutate; + void mutate(Function &F, RandomIRBuilder &IB) override; + void mutate(BasicBlock &BB, RandomIRBuilder &IB) override; +}; + +class InstDeleterIRStrategy : public IRMutationStrategy { +public: + uint64_t getWeight(size_t CurrentSize, size_t MaxSize, + uint64_t CurrentWeight) override; + + using IRMutationStrategy::mutate; + void mutate(Function &F, RandomIRBuilder &IB) override; + void mutate(Instruction &Inst, RandomIRBuilder &IB) override; +}; + +} // end llvm namespace + +#endif // LLVM_FUZZMUTATE_IRMUTATOR_H diff --git a/contrib/llvm/include/llvm/FuzzMutate/OpDescriptor.h b/contrib/llvm/include/llvm/FuzzMutate/OpDescriptor.h new file mode 100644 index 000000000000..e9f8bf09a79b --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/OpDescriptor.h @@ -0,0 +1,219 @@ +//===-- OpDescriptor.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Provides the fuzzerop::Descriptor class and related tools for describing +// operations an IR fuzzer can work with. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_OPDESCRIPTOR_H +#define LLVM_FUZZMUTATE_OPDESCRIPTOR_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include <functional> + +namespace llvm { +namespace fuzzerop { + +/// @{ +/// Populate a small list of potentially interesting constants of a given type. +void makeConstantsWithType(Type *T, std::vector<Constant *> &Cs); +std::vector<Constant *> makeConstantsWithType(Type *T); +/// @} + +/// A matcher/generator for finding suitable values for the next source in an +/// operation's partially completed argument list. +/// +/// Given that we're building some operation X and may have already filled some +/// subset of its operands, this predicate determines if some value New is +/// suitable for the next operand or generates a set of values that are +/// suitable. +class SourcePred { +public: + /// Given a list of already selected operands, returns whether a given new + /// operand is suitable for the next operand. + using PredT = std::function<bool(ArrayRef<Value *> Cur, const Value *New)>; + /// Given a list of already selected operands and a set of valid base types + /// for a fuzzer, generates a list of constants that could be used for the + /// next operand. + using MakeT = std::function<std::vector<Constant *>( + ArrayRef<Value *> Cur, ArrayRef<Type *> BaseTypes)>; + +private: + PredT Pred; + MakeT Make; + +public: + /// Create a fully general source predicate. + SourcePred(PredT Pred, MakeT Make) : Pred(Pred), Make(Make) {} + SourcePred(PredT Pred, NoneType) : Pred(Pred) { + Make = [Pred](ArrayRef<Value *> Cur, ArrayRef<Type *> BaseTypes) { + // Default filter just calls Pred on each of the base types. + std::vector<Constant *> Result; + for (Type *T : BaseTypes) { + Constant *V = UndefValue::get(T); + if (Pred(Cur, V)) + makeConstantsWithType(T, Result); + } + if (Result.empty()) + report_fatal_error("Predicate does not match for base types"); + return Result; + }; + } + + /// Returns true if \c New is compatible for the argument after \c Cur + bool matches(ArrayRef<Value *> Cur, const Value *New) { + return Pred(Cur, New); + } + + /// Generates a list of potential values for the argument after \c Cur. + std::vector<Constant *> generate(ArrayRef<Value *> Cur, + ArrayRef<Type *> BaseTypes) { + return Make(Cur, BaseTypes); + } +}; + +/// A description of some operation we can build while fuzzing IR. +struct OpDescriptor { + unsigned Weight; + SmallVector<SourcePred, 2> SourcePreds; + std::function<Value *(ArrayRef<Value *>, Instruction *)> BuilderFunc; +}; + +static inline SourcePred onlyType(Type *Only) { + auto Pred = [Only](ArrayRef<Value *>, const Value *V) { + return V->getType() == Only; + }; + auto Make = [Only](ArrayRef<Value *>, ArrayRef<Type *>) { + return makeConstantsWithType(Only); + }; + return {Pred, Make}; +} + +static inline SourcePred anyType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + return !V->getType()->isVoidTy(); + }; + auto Make = None; + return {Pred, Make}; +} + +static inline SourcePred anyIntType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + return V->getType()->isIntegerTy(); + }; + auto Make = None; + return {Pred, Make}; +} + +static inline SourcePred anyFloatType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + return V->getType()->isFloatingPointTy(); + }; + auto Make = None; + return {Pred, Make}; +} + +static inline SourcePred anyPtrType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + return V->getType()->isPointerTy(); + }; + auto Make = [](ArrayRef<Value *>, ArrayRef<Type *> Ts) { + std::vector<Constant *> Result; + // TODO: Should these point at something? + for (Type *T : Ts) + Result.push_back(UndefValue::get(PointerType::getUnqual(T))); + return Result; + }; + return {Pred, Make}; +} + +static inline SourcePred sizedPtrType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + if (const auto *PtrT = dyn_cast<PointerType>(V->getType())) + return PtrT->getElementType()->isSized(); + return false; + }; + auto Make = [](ArrayRef<Value *>, ArrayRef<Type *> Ts) { + std::vector<Constant *> Result; + + for (Type *T : Ts) + if (T->isSized()) + Result.push_back(UndefValue::get(PointerType::getUnqual(T))); + + return Result; + }; + return {Pred, Make}; +} + +static inline SourcePred anyAggregateType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + // We can't index zero sized arrays. + if (isa<ArrayType>(V->getType())) + return V->getType()->getArrayNumElements() > 0; + + // Structs can also be zero sized. I.e opaque types. + if (isa<StructType>(V->getType())) + return V->getType()->getStructNumElements() > 0; + + return V->getType()->isAggregateType(); + }; + // TODO: For now we only find aggregates in BaseTypes. It might be better to + // manufacture them out of the base types in some cases. + auto Find = None; + return {Pred, Find}; +} + +static inline SourcePred anyVectorType() { + auto Pred = [](ArrayRef<Value *>, const Value *V) { + return V->getType()->isVectorTy(); + }; + // TODO: For now we only find vectors in BaseTypes. It might be better to + // manufacture vectors out of the base types, but it's tricky to be sure + // that's actually a reasonable type. + auto Make = None; + return {Pred, Make}; +} + +/// Match values that have the same type as the first source. +static inline SourcePred matchFirstType() { + auto Pred = [](ArrayRef<Value *> Cur, const Value *V) { + assert(!Cur.empty() && "No first source yet"); + return V->getType() == Cur[0]->getType(); + }; + auto Make = [](ArrayRef<Value *> Cur, ArrayRef<Type *>) { + assert(!Cur.empty() && "No first source yet"); + return makeConstantsWithType(Cur[0]->getType()); + }; + return {Pred, Make}; +} + +/// Match values that have the first source's scalar type. +static inline SourcePred matchScalarOfFirstType() { + auto Pred = [](ArrayRef<Value *> Cur, const Value *V) { + assert(!Cur.empty() && "No first source yet"); + return V->getType() == Cur[0]->getType()->getScalarType(); + }; + auto Make = [](ArrayRef<Value *> Cur, ArrayRef<Type *>) { + assert(!Cur.empty() && "No first source yet"); + return makeConstantsWithType(Cur[0]->getType()->getScalarType()); + }; + return {Pred, Make}; +} + +} // end fuzzerop namespace +} // end llvm namespace + +#endif // LLVM_FUZZMUTATE_OPDESCRIPTOR_H diff --git a/contrib/llvm/include/llvm/FuzzMutate/Operations.h b/contrib/llvm/include/llvm/FuzzMutate/Operations.h new file mode 100644 index 000000000000..668bd952ebb2 --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/Operations.h @@ -0,0 +1,54 @@ +//===-- Operations.h - ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementations of common fuzzer operation descriptors for building an IR +// mutator. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_OPERATIONS_H +#define LLVM_FUZZMUTATE_OPERATIONS_H + +#include "llvm/FuzzMutate/OpDescriptor.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" + +namespace llvm { + +/// Getters for the default sets of operations, per general category. +/// @{ +void describeFuzzerIntOps(std::vector<fuzzerop::OpDescriptor> &Ops); +void describeFuzzerFloatOps(std::vector<fuzzerop::OpDescriptor> &Ops); +void describeFuzzerControlFlowOps(std::vector<fuzzerop::OpDescriptor> &Ops); +void describeFuzzerPointerOps(std::vector<fuzzerop::OpDescriptor> &Ops); +void describeFuzzerAggregateOps(std::vector<fuzzerop::OpDescriptor> &Ops); +void describeFuzzerVectorOps(std::vector<fuzzerop::OpDescriptor> &Ops); +/// @} + +namespace fuzzerop { + +/// Descriptors for individual operations. +/// @{ +OpDescriptor binOpDescriptor(unsigned Weight, Instruction::BinaryOps Op); +OpDescriptor cmpOpDescriptor(unsigned Weight, Instruction::OtherOps CmpOp, + CmpInst::Predicate Pred); +OpDescriptor splitBlockDescriptor(unsigned Weight); +OpDescriptor gepDescriptor(unsigned Weight); +OpDescriptor extractValueDescriptor(unsigned Weight); +OpDescriptor insertValueDescriptor(unsigned Weight); +OpDescriptor extractElementDescriptor(unsigned Weight); +OpDescriptor insertElementDescriptor(unsigned Weight); +OpDescriptor shuffleVectorDescriptor(unsigned Weight); +/// @} + +} // end fuzzerop namespace + +} // end llvm namespace + +#endif // LLVM_FUZZMUTATE_OPERATIONS_H diff --git a/contrib/llvm/include/llvm/FuzzMutate/Random.h b/contrib/llvm/include/llvm/FuzzMutate/Random.h new file mode 100644 index 000000000000..3a5f46a07554 --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/Random.h @@ -0,0 +1,97 @@ +//===--- Random.h - Utilities for random sampling -------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Utilities for random sampling. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_RANDOM_H +#define LLVM_FUZZMUTATE_RANDOM_H + +#include <random> +#include "llvm/Support/raw_ostream.h" +namespace llvm { + +/// Return a uniformly distributed random value between \c Min and \c Max +template <typename T, typename GenT> T uniform(GenT &Gen, T Min, T Max) { + return std::uniform_int_distribution<T>(Min, Max)(Gen); +} + +/// Return a uniformly distributed random value of type \c T +template <typename T, typename GenT> T uniform(GenT &Gen) { + return uniform<T>(Gen, std::numeric_limits<T>::min(), + std::numeric_limits<T>::max()); +} + +/// Randomly selects an item by sampling into a set with an unknown number of +/// elements, which may each be weighted to be more likely choices. +template <typename T, typename GenT> class ReservoirSampler { + GenT &RandGen; + typename std::remove_const<T>::type Selection = {}; + uint64_t TotalWeight = 0; + +public: + ReservoirSampler(GenT &RandGen) : RandGen(RandGen) {} + + uint64_t totalWeight() const { return TotalWeight; } + bool isEmpty() const { return TotalWeight == 0; } + + const T &getSelection() const { + assert(!isEmpty() && "Nothing selected"); + return Selection; + } + + explicit operator bool() const { return !isEmpty();} + const T &operator*() const { return getSelection(); } + + /// Sample each item in \c Items with unit weight + template <typename RangeT> ReservoirSampler &sample(RangeT &&Items) { + for (auto &I : Items) + sample(I, 1); + return *this; + } + + /// Sample a single item with the given weight. + ReservoirSampler &sample(const T &Item, uint64_t Weight) { + if (!Weight) + // If the weight is zero, do nothing. + return *this; + TotalWeight += Weight; + // Consider switching from the current element to this one. + if (uniform<uint64_t>(RandGen, 1, TotalWeight) <= Weight) + Selection = Item; + return *this; + } +}; + +template <typename GenT, typename RangeT, + typename ElT = typename std::remove_reference< + decltype(*std::begin(std::declval<RangeT>()))>::type> +ReservoirSampler<ElT, GenT> makeSampler(GenT &RandGen, RangeT &&Items) { + ReservoirSampler<ElT, GenT> RS(RandGen); + RS.sample(Items); + return RS; +} + +template <typename GenT, typename T> +ReservoirSampler<T, GenT> makeSampler(GenT &RandGen, const T &Item, + uint64_t Weight) { + ReservoirSampler<T, GenT> RS(RandGen); + RS.sample(Item, Weight); + return RS; +} + +template <typename T, typename GenT> +ReservoirSampler<T, GenT> makeSampler(GenT &RandGen) { + return ReservoirSampler<T, GenT>(RandGen); +} + +} // End llvm namespace + +#endif // LLVM_FUZZMUTATE_RANDOM_H diff --git a/contrib/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h b/contrib/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h new file mode 100644 index 000000000000..5cf3f0b22709 --- /dev/null +++ b/contrib/llvm/include/llvm/FuzzMutate/RandomIRBuilder.h @@ -0,0 +1,62 @@ +//===-- Mutator.h - Utils for randomly mutation IR --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Provides the Mutator class, which is used to mutate IR for fuzzing. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZMUTATE_RANDOMIRBUILDER_H +#define LLVM_FUZZMUTATE_RANDOMIRBUILDER_H + +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/FuzzMutate/IRMutator.h" +#include "llvm/FuzzMutate/Random.h" +#include <random> + +namespace llvm { + +using RandomEngine = std::mt19937; + +struct RandomIRBuilder { + RandomEngine Rand; + SmallVector<Type *, 16> KnownTypes; + + RandomIRBuilder(int Seed, ArrayRef<Type *> AllowedTypes) + : Rand(Seed), KnownTypes(AllowedTypes.begin(), AllowedTypes.end()) {} + + // TODO: Try to make this a bit less of a random mishmash of functions. + + /// Find a "source" for some operation, which will be used in one of the + /// operation's operands. This either selects an instruction in \c Insts or + /// returns some new arbitrary Value. + Value *findOrCreateSource(BasicBlock &BB, ArrayRef<Instruction *> Insts); + /// Find a "source" for some operation, which will be used in one of the + /// operation's operands. This either selects an instruction in \c Insts that + /// matches \c Pred, or returns some new Value that matches \c Pred. The + /// values in \c Srcs should be source operands that have already been + /// selected. + Value *findOrCreateSource(BasicBlock &BB, ArrayRef<Instruction *> Insts, + ArrayRef<Value *> Srcs, fuzzerop::SourcePred Pred); + /// Create some Value suitable as a source for some operation. + Value *newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts, + ArrayRef<Value *> Srcs, fuzzerop::SourcePred Pred); + /// Find a viable user for \c V in \c Insts, which should all be contained in + /// \c BB. This may also create some new instruction in \c BB and use that. + void connectToSink(BasicBlock &BB, ArrayRef<Instruction *> Insts, Value *V); + /// Create a user for \c V in \c BB. + void newSink(BasicBlock &BB, ArrayRef<Instruction *> Insts, Value *V); + Value *findPointer(BasicBlock &BB, ArrayRef<Instruction *> Insts, + ArrayRef<Value *> Srcs, fuzzerop::SourcePred Pred); + Type *chooseType(LLVMContext &Context, ArrayRef<Value *> Srcs, + fuzzerop::SourcePred Pred); +}; + +} // end llvm namespace + +#endif // LLVM_FUZZMUTATE_RANDOMIRBUILDER_H diff --git a/contrib/llvm/include/llvm/IR/Attributes.h b/contrib/llvm/include/llvm/IR/Attributes.h index 0cab8bbb8ead..a05a01073049 100644 --- a/contrib/llvm/include/llvm/IR/Attributes.h +++ b/contrib/llvm/include/llvm/IR/Attributes.h @@ -225,8 +225,8 @@ public: static AttributeSet get(LLVMContext &C, const AttrBuilder &B); static AttributeSet get(LLVMContext &C, ArrayRef<Attribute> Attrs); - bool operator==(const AttributeSet &O) { return SetNode == O.SetNode; } - bool operator!=(const AttributeSet &O) { return !(*this == O); } + bool operator==(const AttributeSet &O) const { return SetNode == O.SetNode; } + bool operator!=(const AttributeSet &O) const { return !(*this == O); } /// Add an argument attribute. Returns a new set because attribute sets are /// immutable. diff --git a/contrib/llvm/include/llvm/IR/Attributes.td b/contrib/llvm/include/llvm/IR/Attributes.td index 616387816bf8..ebe5c1985875 100644 --- a/contrib/llvm/include/llvm/IR/Attributes.td +++ b/contrib/llvm/include/llvm/IR/Attributes.td @@ -149,6 +149,9 @@ def StackProtectReq : EnumAttr<"sspreq">; /// Strong Stack protection. def StackProtectStrong : EnumAttr<"sspstrong">; +/// Function was called in a scope requiring strict floating point semantics. +def StrictFP : EnumAttr<"strictfp">; + /// Hidden pointer to structure to return. def StructRet : EnumAttr<"sret">; @@ -161,6 +164,9 @@ def SanitizeThread : EnumAttr<"sanitize_thread">; /// MemorySanitizer is on. def SanitizeMemory : EnumAttr<"sanitize_memory">; +/// HWAddressSanitizer is on. +def SanitizeHWAddress : EnumAttr<"sanitize_hwaddress">; + /// Argument is swift error. def SwiftError : EnumAttr<"swifterror">; @@ -182,6 +188,7 @@ def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; def NoJumpTables : StrBoolAttr<"no-jump-tables">; +def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">; class CompatRule<string F> { // The name of the function called to check the attribute of the caller and @@ -196,6 +203,7 @@ class CompatRule<string F> { def : CompatRule<"isEqual<SanitizeAddressAttr>">; def : CompatRule<"isEqual<SanitizeThreadAttr>">; def : CompatRule<"isEqual<SanitizeMemoryAttr>">; +def : CompatRule<"isEqual<SanitizeHWAddressAttr>">; def : CompatRule<"isEqual<SafeStackAttr>">; class MergeRule<string F> { @@ -213,6 +221,7 @@ def : MergeRule<"setAND<NoNansFPMathAttr>">; def : MergeRule<"setAND<UnsafeFPMathAttr>">; def : MergeRule<"setOR<NoImplicitFloatAttr>">; def : MergeRule<"setOR<NoJumpTablesAttr>">; +def : MergeRule<"setOR<ProfileSampleAccurateAttr>">; def : MergeRule<"adjustCallerSSPLevel">; def : MergeRule<"adjustCallerStackProbes">; def : MergeRule<"adjustCallerStackProbeSize">; diff --git a/contrib/llvm/include/llvm/IR/BasicBlock.h b/contrib/llvm/include/llvm/IR/BasicBlock.h index 6714f2c97473..77cfc9776df0 100644 --- a/contrib/llvm/include/llvm/IR/BasicBlock.h +++ b/contrib/llvm/include/llvm/IR/BasicBlock.h @@ -398,6 +398,8 @@ public: /// \brief Return true if it is legal to hoist instructions into this block. bool isLegalToHoistInto() const; + Optional<uint64_t> getIrrLoopHeaderWeight() const; + private: /// \brief Increment the internal refcount of the number of BlockAddresses /// referencing this BasicBlock by \p Amt. diff --git a/contrib/llvm/include/llvm/IR/CallSite.h b/contrib/llvm/include/llvm/IR/CallSite.h index 96fbebf42c38..5b10da8f2aee 100644 --- a/contrib/llvm/include/llvm/IR/CallSite.h +++ b/contrib/llvm/include/llvm/IR/CallSite.h @@ -35,7 +35,6 @@ #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Use.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" @@ -46,6 +45,10 @@ namespace llvm { +namespace Intrinsic { +enum ID : unsigned; +} + template <typename FunTy = const Function, typename BBTy = const BasicBlock, typename ValTy = const Value, @@ -59,7 +62,7 @@ class CallSiteBase { protected: PointerIntPair<InstrTy*, 1, bool> I; - CallSiteBase() : I(nullptr, false) {} + CallSiteBase() = default; CallSiteBase(CallTy *CI) : I(CI, true) { assert(CI); } CallSiteBase(InvokeTy *II) : I(II, false) { assert(II); } explicit CallSiteBase(ValTy *II) { *this = get(II); } @@ -107,12 +110,12 @@ public: /// Return true if the callsite is an indirect call. bool isIndirectCall() const { - Value *V = getCalledValue(); + const Value *V = getCalledValue(); if (!V) return false; if (isa<FunTy>(V) || isa<Constant>(V)) return false; - if (CallInst *CI = dyn_cast<CallInst>(getInstruction())) { + if (const CallInst *CI = dyn_cast<CallInst>(getInstruction())) { if (CI->isInlineAsm()) return false; } @@ -426,6 +429,11 @@ public: CALLSITE_DELEGATE_GETTER(isNoBuiltin()); } + /// Return true if the call requires strict floating point semantics. + bool isStrictFP() const { + CALLSITE_DELEGATE_GETTER(isStrictFP()); + } + /// Return true if the call should not be inlined. bool isNoInline() const { CALLSITE_DELEGATE_GETTER(isNoInline()); @@ -467,6 +475,24 @@ public: CALLSITE_DELEGATE_SETTER(setOnlyAccessesArgMemory()); } + /// Determine if the function may only access memory that is + /// inaccessible from the IR. + bool onlyAccessesInaccessibleMemory() const { + CALLSITE_DELEGATE_GETTER(onlyAccessesInaccessibleMemory()); + } + void setOnlyAccessesInaccessibleMemory() { + CALLSITE_DELEGATE_SETTER(setOnlyAccessesInaccessibleMemory()); + } + + /// Determine if the function may only access memory that is + /// either inaccessible from the IR or pointed to by its arguments. + bool onlyAccessesInaccessibleMemOrArgMem() const { + CALLSITE_DELEGATE_GETTER(onlyAccessesInaccessibleMemOrArgMem()); + } + void setOnlyAccessesInaccessibleMemOrArgMem() { + CALLSITE_DELEGATE_SETTER(setOnlyAccessesInaccessibleMemOrArgMem()); + } + /// Determine if the call cannot return. bool doesNotReturn() const { CALLSITE_DELEGATE_GETTER(doesNotReturn()); @@ -488,7 +514,7 @@ public: CALLSITE_DELEGATE_GETTER(cannotDuplicate()); } void setCannotDuplicate() { - CALLSITE_DELEGATE_GETTER(setCannotDuplicate()); + CALLSITE_DELEGATE_SETTER(setCannotDuplicate()); } /// Determine if the call is convergent. diff --git a/contrib/llvm/include/llvm/IR/CallingConv.h b/contrib/llvm/include/llvm/IR/CallingConv.h index 850964afc307..84fe836adc35 100644 --- a/contrib/llvm/include/llvm/IR/CallingConv.h +++ b/contrib/llvm/include/llvm/IR/CallingConv.h @@ -183,16 +183,18 @@ namespace CallingConv { /// which have an "optimized" convention to preserve registers. AVR_BUILTIN = 86, - /// Calling convention used for Mesa vertex shaders. + /// Calling convention used for Mesa vertex shaders, or AMDPAL last shader + /// stage before rasterization (vertex shader if tessellation and geometry + /// are not in use, or otherwise copy shader if one is needed). AMDGPU_VS = 87, - /// Calling convention used for Mesa geometry shaders. + /// Calling convention used for Mesa/AMDPAL geometry shaders. AMDGPU_GS = 88, - /// Calling convention used for Mesa pixel shaders. + /// Calling convention used for Mesa/AMDPAL pixel shaders. AMDGPU_PS = 89, - /// Calling convention used for Mesa compute shaders. + /// Calling convention used for Mesa/AMDPAL compute shaders. AMDGPU_CS = 90, /// Calling convention for AMDGPU code object kernels. @@ -201,14 +203,23 @@ namespace CallingConv { /// Register calling convention used for parameters transfer optimization X86_RegCall = 92, - /// Calling convention used for Mesa hull shaders. (= tessellation control - /// shaders) + /// Calling convention used for Mesa/AMDPAL hull shaders (= tessellation + /// control shaders). AMDGPU_HS = 93, /// Calling convention used for special MSP430 rtlib functions /// which have an "optimized" convention using additional registers. MSP430_BUILTIN = 94, + /// Calling convention used for AMDPAL vertex shader if tessellation is in + /// use. + AMDGPU_LS = 95, + + /// Calling convention used for AMDPAL shader stage before geometry shader + /// if geometry is in use. So either the domain (= tessellation evaluation) + /// shader if tessellation is in use, or otherwise the vertex shader. + AMDGPU_ES = 96, + /// The highest possible calling convention ID. Must be some 2^k - 1. MaxID = 1023 }; diff --git a/contrib/llvm/include/llvm/IR/Constant.h b/contrib/llvm/include/llvm/IR/Constant.h index 9daeac6ad6e7..0c94b58a3112 100644 --- a/contrib/llvm/include/llvm/IR/Constant.h +++ b/contrib/llvm/include/llvm/IR/Constant.h @@ -117,8 +117,8 @@ public: //// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { - return V->getValueID() >= ConstantFirstVal && - V->getValueID() <= ConstantLastVal; + static_assert(ConstantFirstVal == 0, "V->getValueID() >= ConstantFirstVal always succeeds"); + return V->getValueID() <= ConstantLastVal; } /// This method is a special form of User::replaceUsesOfWith diff --git a/contrib/llvm/include/llvm/IR/ConstantRange.h b/contrib/llvm/include/llvm/IR/ConstantRange.h index ff6495e7f075..6889e2658244 100644 --- a/contrib/llvm/include/llvm/IR/ConstantRange.h +++ b/contrib/llvm/include/llvm/IR/ConstantRange.h @@ -96,9 +96,9 @@ public: /// /// NB! The returned set does *not* contain **all** possible values of X for /// which "X BinOpC Y" does not wrap -- some viable values of X may be - /// missing, so you cannot use this to constrain X's range. E.g. in the last - /// example, "(-2) + 1" is both nsw and nuw (so the "X" could be -2), but (-2) - /// is not in the set returned. + /// missing, so you cannot use this to constrain X's range. E.g. in the + /// fourth example, "(-2) + 1" is both nsw and nuw (so the "X" could be -2), + /// but (-2) is not in the set returned. /// /// Examples: /// typedef OverflowingBinaryOperator OBO; @@ -109,6 +109,10 @@ public: /// MGNR(Add, [i8 1, 2), OBO::NoUnsignedWrap | OBO::NoSignedWrap) /// == [0,INT_MAX) /// MGNR(Add, [i8 -1, 6), OBO::NoSignedWrap) == [INT_MIN+1, INT_MAX-4) + /// MGNR(Sub, [i8 1, 2), OBO::NoSignedWrap) == [-127, 128) + /// MGNR(Sub, [i8 1, 2), OBO::NoUnsignedWrap) == [1, 0) + /// MGNR(Sub, [i8 1, 2), OBO::NoUnsignedWrap | OBO::NoSignedWrap) + /// == [1,INT_MAX) static ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind); @@ -313,6 +317,10 @@ public: /// logical right shift of a value in this range and a value in \p Other. ConstantRange lshr(const ConstantRange &Other) const; + /// Return a new range representing the possible values resulting from a + /// arithmetic right shift of a value in this range and a value in \p Other. + ConstantRange ashr(const ConstantRange &Other) const; + /// Return a new range that is the logical not of the current set. ConstantRange inverse() const; diff --git a/contrib/llvm/include/llvm/IR/DIBuilder.h b/contrib/llvm/include/llvm/IR/DIBuilder.h index 6a14f783005d..3c2074dfe788 100644 --- a/contrib/llvm/include/llvm/IR/DIBuilder.h +++ b/contrib/llvm/include/llvm/IR/DIBuilder.h @@ -74,6 +74,17 @@ namespace llvm { /// Create an \a temporary node and track it in \a UnresolvedNodes. void trackIfUnresolved(MDNode *N); + /// Internal helper for insertDeclare. + Instruction *insertDeclare(llvm::Value *Storage, DILocalVariable *VarInfo, + DIExpression *Expr, const DILocation *DL, + BasicBlock *InsertBB, Instruction *InsertBefore); + + /// Internal helper for insertDbgValueIntrinsic. + Instruction * + insertDbgValueIntrinsic(llvm::Value *Val, DILocalVariable *VarInfo, + DIExpression *Expr, const DILocation *DL, + BasicBlock *InsertBB, Instruction *InsertBefore); + public: /// Construct a builder for a module. /// @@ -112,6 +123,8 @@ namespace llvm { /// \param SplitDebugInlining Whether to emit inline debug info. /// \param DebugInfoForProfiling Whether to emit extra debug info for /// profile collection. + /// \param GnuPubnames Whether to emit .debug_gnu_pubnames section instead + /// of .debug_pubnames. DICompileUnit * createCompileUnit(unsigned Lang, DIFile *File, StringRef Producer, bool isOptimized, StringRef Flags, unsigned RV, @@ -119,7 +132,8 @@ namespace llvm { DICompileUnit::DebugEmissionKind Kind = DICompileUnit::DebugEmissionKind::FullDebug, uint64_t DWOId = 0, bool SplitDebugInlining = true, - bool DebugInfoForProfiling = false); + bool DebugInfoForProfiling = false, + bool GnuPubnames = false); /// Create a file descriptor to hold debugging information for a file. /// \param Filename File name. @@ -551,14 +565,6 @@ namespace llvm { DIExpression *createExpression(ArrayRef<uint64_t> Addr = None); DIExpression *createExpression(ArrayRef<int64_t> Addr); - /// Create a descriptor to describe one part - /// of aggregate variable that is fragmented across multiple Values. - /// - /// \param OffsetInBits Offset of the piece in bits. - /// \param SizeInBits Size of the piece in bits. - DIExpression *createFragmentExpression(unsigned OffsetInBits, - unsigned SizeInBits); - /// Create an expression for a variable that does not have an address, but /// does have a constant value. DIExpression *createConstantValueExpression(uint64_t Val) { @@ -729,12 +735,11 @@ namespace llvm { /// Insert a new llvm.dbg.value intrinsic call. /// \param Val llvm::Value of the variable - /// \param Offset Offset /// \param VarInfo Variable's debug info descriptor. /// \param Expr A complex location expression. /// \param DL Debug info location. /// \param InsertAtEnd Location for the new intrinsic. - Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset, + Instruction *insertDbgValueIntrinsic(llvm::Value *Val, DILocalVariable *VarInfo, DIExpression *Expr, const DILocation *DL, @@ -742,23 +747,22 @@ namespace llvm { /// Insert a new llvm.dbg.value intrinsic call. /// \param Val llvm::Value of the variable - /// \param Offset Offset /// \param VarInfo Variable's debug info descriptor. /// \param Expr A complex location expression. /// \param DL Debug info location. /// \param InsertBefore Location for the new intrinsic. - Instruction *insertDbgValueIntrinsic(llvm::Value *Val, uint64_t Offset, + Instruction *insertDbgValueIntrinsic(llvm::Value *Val, DILocalVariable *VarInfo, DIExpression *Expr, const DILocation *DL, Instruction *InsertBefore); - /// Replace the vtable holder in the given composite type. + /// Replace the vtable holder in the given type. /// /// If this creates a self reference, it may orphan some unresolved cycles /// in the operands of \c T, so \a DIBuilder needs to track that. void replaceVTableHolder(DICompositeType *&T, - DICompositeType *VTableHolder); + DIType *VTableHolder); /// Replace arrays on a composite type. /// diff --git a/contrib/llvm/include/llvm/IR/DataLayout.h b/contrib/llvm/include/llvm/IR/DataLayout.h index daf8f8da689d..a6c71a5a2c3e 100644 --- a/contrib/llvm/include/llvm/IR/DataLayout.h +++ b/contrib/llvm/include/llvm/IR/DataLayout.h @@ -309,9 +309,7 @@ public: } /// Layout pointer alignment - /// FIXME: The defaults need to be removed once all of - /// the backends/clients are updated. - unsigned getPointerABIAlignment(unsigned AS = 0) const; + unsigned getPointerABIAlignment(unsigned AS) const; /// Return target's alignment for stack-based pointers /// FIXME: The defaults need to be removed once all of diff --git a/contrib/llvm/include/llvm/IR/DebugInfoMetadata.h b/contrib/llvm/include/llvm/IR/DebugInfoMetadata.h index 678a43ae7926..75b0c43b6512 100644 --- a/contrib/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/contrib/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -473,10 +473,12 @@ class DIFile : public DIScope { friend class MDNode; public: + // These values must be explictly set, as they end up in the final object + // file. enum ChecksumKind { - CSK_None, - CSK_MD5, - CSK_SHA1, + CSK_None = 0, + CSK_MD5 = 1, + CSK_SHA1 = 2, CSK_Last = CSK_SHA1 // Should be last enumeration. }; @@ -510,7 +512,7 @@ public: ChecksumKind CSK = CSK_None, StringRef CS = StringRef()), (Filename, Directory, CSK, CS)) - DEFINE_MDNODE_GET(DIFile, (MDString *Filename, MDString *Directory, + DEFINE_MDNODE_GET(DIFile, (MDString * Filename, MDString *Directory, ChecksumKind CSK = CSK_None, MDString *CS = nullptr), (Filename, Directory, CSK, CS)) @@ -1068,16 +1070,17 @@ private: uint64_t DWOId; bool SplitDebugInlining; bool DebugInfoForProfiling; + bool GnuPubnames; DICompileUnit(LLVMContext &C, StorageType Storage, unsigned SourceLanguage, bool IsOptimized, unsigned RuntimeVersion, unsigned EmissionKind, uint64_t DWOId, bool SplitDebugInlining, - bool DebugInfoForProfiling, ArrayRef<Metadata *> Ops) + bool DebugInfoForProfiling, bool GnuPubnames, ArrayRef<Metadata *> Ops) : DIScope(C, DICompileUnitKind, Storage, dwarf::DW_TAG_compile_unit, Ops), SourceLanguage(SourceLanguage), IsOptimized(IsOptimized), RuntimeVersion(RuntimeVersion), EmissionKind(EmissionKind), DWOId(DWOId), SplitDebugInlining(SplitDebugInlining), - DebugInfoForProfiling(DebugInfoForProfiling) { + DebugInfoForProfiling(DebugInfoForProfiling), GnuPubnames(GnuPubnames) { assert(Storage != Uniqued); } ~DICompileUnit() = default; @@ -1091,15 +1094,14 @@ private: DIGlobalVariableExpressionArray GlobalVariables, DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling, - StorageType Storage, bool ShouldCreate = true) { - return getImpl(Context, SourceLanguage, File, - getCanonicalMDString(Context, Producer), IsOptimized, - getCanonicalMDString(Context, Flags), RuntimeVersion, - getCanonicalMDString(Context, SplitDebugFilename), - EmissionKind, EnumTypes.get(), RetainedTypes.get(), - GlobalVariables.get(), ImportedEntities.get(), Macros.get(), - DWOId, SplitDebugInlining, DebugInfoForProfiling, Storage, - ShouldCreate); + bool GnuPubnames, StorageType Storage, bool ShouldCreate = true) { + return getImpl( + Context, SourceLanguage, File, getCanonicalMDString(Context, Producer), + IsOptimized, getCanonicalMDString(Context, Flags), RuntimeVersion, + getCanonicalMDString(Context, SplitDebugFilename), EmissionKind, + EnumTypes.get(), RetainedTypes.get(), GlobalVariables.get(), + ImportedEntities.get(), Macros.get(), DWOId, SplitDebugInlining, + DebugInfoForProfiling, GnuPubnames, Storage, ShouldCreate); } static DICompileUnit * getImpl(LLVMContext &Context, unsigned SourceLanguage, Metadata *File, @@ -1108,7 +1110,7 @@ private: unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, bool SplitDebugInlining, - bool DebugInfoForProfiling, StorageType Storage, + bool DebugInfoForProfiling, bool GnuPubnames, StorageType Storage, bool ShouldCreate = true); TempDICompileUnit cloneImpl() const { @@ -1118,7 +1120,7 @@ private: getEmissionKind(), getEnumTypes(), getRetainedTypes(), getGlobalVariables(), getImportedEntities(), getMacros(), DWOId, getSplitDebugInlining(), - getDebugInfoForProfiling()); + getDebugInfoForProfiling(), getGnuPubnames()); } public: @@ -1133,11 +1135,12 @@ public: DICompositeTypeArray EnumTypes, DIScopeArray RetainedTypes, DIGlobalVariableExpressionArray GlobalVariables, DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, - uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling), + uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling, + bool GnuPubnames), (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, Macros, DWOId, SplitDebugInlining, - DebugInfoForProfiling)) + DebugInfoForProfiling, GnuPubnames)) DEFINE_MDNODE_GET_DISTINCT_TEMPORARY( DICompileUnit, (unsigned SourceLanguage, Metadata *File, MDString *Producer, @@ -1145,11 +1148,11 @@ public: MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, - bool SplitDebugInlining, bool DebugInfoForProfiling), + bool SplitDebugInlining, bool DebugInfoForProfiling, bool GnuPubnames), (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, Macros, DWOId, SplitDebugInlining, - DebugInfoForProfiling)) + DebugInfoForProfiling, GnuPubnames)) TempDICompileUnit clone() const { return cloneImpl(); } @@ -1160,6 +1163,7 @@ public: return (DebugEmissionKind)EmissionKind; } bool getDebugInfoForProfiling() const { return DebugInfoForProfiling; } + bool getGnuPubnames() const { return GnuPubnames; } StringRef getProducer() const { return getStringOperand(1); } StringRef getFlags() const { return getStringOperand(2); } StringRef getSplitDebugFilename() const { return getStringOperand(3); } @@ -1413,17 +1417,17 @@ public: /// could create a location with a new discriminator. If they are from /// different files/lines the location is ambiguous and can't be /// represented in a single line entry. In this case, no location - /// should be set. + /// should be set, unless the merged instruction is a call, which we will + /// set the merged debug location as line 0 of the nearest common scope + /// where 2 locations are inlined from. This only applies to Instruction; + /// for MachineInstruction, as it is post-inline, we will treat the call + /// instruction the same way as other instructions. /// - /// Currently the function does not create a new location. If the locations - /// are the same, or cannot be discriminated, the first location is returned. - /// Otherwise an empty location will be used. - static const DILocation *getMergedLocation(const DILocation *LocA, - const DILocation *LocB) { - if (LocA && LocB && (LocA == LocB || !LocA->canDiscriminate(*LocB))) - return LocA; - return nullptr; - } + /// \p ForInst: The Instruction the merged DILocation is for. If the + /// Instruction is unavailable or non-existent, use nullptr. + static const DILocation * + getMergedLocation(const DILocation *LocA, const DILocation *LocB, + const Instruction *ForInst = nullptr); /// Returns the base discriminator for a given encoded discriminator \p D. static unsigned getBaseDiscriminatorFromDiscriminator(unsigned D) { @@ -2087,6 +2091,8 @@ public: DITypeRef getType() const { return DITypeRef(getRawType()); } uint32_t getAlignInBits() const { return AlignInBits; } uint32_t getAlignInBytes() const { return getAlignInBits() / CHAR_BIT; } + /// Determines the size of the variable's type. + Optional<uint64_t> getSizeInBits() const; StringRef getFilename() const { if (auto *F = getFile()) @@ -2291,8 +2297,23 @@ public: /// Prepend \p DIExpr with a deref and offset operation and optionally turn it /// into a stack value. - static DIExpression *prepend(const DIExpression *DIExpr, bool Deref, - int64_t Offset = 0, bool StackValue = false); + static DIExpression *prepend(const DIExpression *DIExpr, bool DerefBefore, + int64_t Offset = 0, bool DerefAfter = false, + bool StackValue = false); + + /// Create a DIExpression to describe one part of an aggregate variable that + /// is fragmented across multiple Values. The DW_OP_LLVM_fragment operation + /// will be appended to the elements of \c Expr. If \c Expr already contains + /// a \c DW_OP_LLVM_fragment \c OffsetInBits is interpreted as an offset + /// into the existing fragment. + /// + /// \param OffsetInBits Offset of the piece in bits. + /// \param SizeInBits Size of the piece in bits. + /// \return Creating a fragment expression may fail if \c Expr + /// contains arithmetic operations that would be truncated. + static Optional<DIExpression *> + createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, + unsigned SizeInBits); }; /// Global variables. @@ -2630,7 +2651,7 @@ public: Metadata *getRawExpression() const { return getOperand(1); } DIExpression *getExpression() const { - return cast_or_null<DIExpression>(getRawExpression()); + return cast<DIExpression>(getRawExpression()); } static bool classof(const Metadata *MD) { diff --git a/contrib/llvm/include/llvm/IR/DiagnosticHandler.h b/contrib/llvm/include/llvm/IR/DiagnosticHandler.h new file mode 100644 index 000000000000..9256d4850df1 --- /dev/null +++ b/contrib/llvm/include/llvm/IR/DiagnosticHandler.h @@ -0,0 +1,75 @@ +//===- DiagnosticHandler.h - DiagnosticHandler class for LLVM -*- C++ ---*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Base DiagnosticHandler class declaration. Derive from this class to provide +// custom diagnostic reporting. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_DIAGNOSTICHANDLER_H +#define LLVM_IR_DIAGNOSTICHANDLER_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { +class DiagnosticInfo; + +/// \brief This is the base class for diagnostic handling in LLVM. +/// The handleDiagnostics method must be overriden by the subclasses to handle +/// diagnostic. The *RemarkEnabled methods can be overriden to control +/// which remarks are enabled. +struct DiagnosticHandler { + void *DiagnosticContext = nullptr; + DiagnosticHandler(void *DiagContext = nullptr) + : DiagnosticContext(DiagContext) {} + virtual ~DiagnosticHandler() = default; + + using DiagnosticHandlerTy = void (*)(const DiagnosticInfo &DI, void *Context); + + /// DiagHandlerCallback is settable from the C API and base implementation + /// of DiagnosticHandler will call it from handleDiagnostics(). Any derived + /// class of DiagnosticHandler should not use callback but + /// implement handleDiagnostics(). + DiagnosticHandlerTy DiagHandlerCallback = nullptr; + + /// Override handleDiagnostics to provide custom implementation. + /// Return true if it handles diagnostics reporting properly otherwise + /// return false to make LLVMContext::diagnose() to print the message + /// with a prefix based on the severity. + virtual bool handleDiagnostics(const DiagnosticInfo &DI) { + if (DiagHandlerCallback) { + DiagHandlerCallback(DI, DiagnosticContext); + return true; + } + return false; + } + + /// Return true if analysis remarks are enabled, override + /// to provide different implementation. + virtual bool isAnalysisRemarkEnabled(StringRef PassName) const; + + /// Return true if missed optimization remarks are enabled, override + /// to provide different implementation. + virtual bool isMissedOptRemarkEnabled(StringRef PassName) const; + + /// Return true if passed optimization remarks are enabled, override + /// to provide different implementation. + virtual bool isPassedOptRemarkEnabled(StringRef PassName) const; + + /// Return true if any type of remarks are enabled for this pass. + bool isAnyRemarkEnabled(StringRef PassName) const { + return (isMissedOptRemarkEnabled(PassName) || + isPassedOptRemarkEnabled(PassName) || + isAnalysisRemarkEnabled(PassName)); + } + + /// Return true if any type of remarks are enabled for any pass. + virtual bool isAnyRemarkEnabled() const; +}; +} // namespace llvm + +#endif // LLVM_IR_DIAGNOSTICHANDLER_H diff --git a/contrib/llvm/include/llvm/IR/DiagnosticInfo.h b/contrib/llvm/include/llvm/IR/DiagnosticInfo.h index 15d332577113..020b67d6b711 100644 --- a/contrib/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/contrib/llvm/include/llvm/IR/DiagnosticInfo.h @@ -188,17 +188,13 @@ private: public: /// \p The function that is concerned by this stack size diagnostic. /// \p The computed stack size. - DiagnosticInfoResourceLimit(const Function &Fn, - const char *ResourceName, + DiagnosticInfoResourceLimit(const Function &Fn, const char *ResourceName, uint64_t ResourceSize, DiagnosticSeverity Severity = DS_Warning, DiagnosticKind Kind = DK_ResourceLimit, uint64_t ResourceLimit = 0) - : DiagnosticInfo(Kind, Severity), - Fn(Fn), - ResourceName(ResourceName), - ResourceSize(ResourceSize), - ResourceLimit(ResourceLimit) {} + : DiagnosticInfo(Kind, Severity), Fn(Fn), ResourceName(ResourceName), + ResourceSize(ResourceSize), ResourceLimit(ResourceLimit) {} const Function &getFunction() const { return Fn; } const char *getResourceName() const { return ResourceName; } @@ -209,19 +205,17 @@ public: void print(DiagnosticPrinter &DP) const override; static bool classof(const DiagnosticInfo *DI) { - return DI->getKind() == DK_ResourceLimit || - DI->getKind() == DK_StackSize; + return DI->getKind() == DK_ResourceLimit || DI->getKind() == DK_StackSize; } }; class DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit { public: - DiagnosticInfoStackSize(const Function &Fn, - uint64_t StackSize, + DiagnosticInfoStackSize(const Function &Fn, uint64_t StackSize, DiagnosticSeverity Severity = DS_Warning, uint64_t StackLimit = 0) - : DiagnosticInfoResourceLimit(Fn, "stack size", StackSize, - Severity, DK_StackSize, StackLimit) {} + : DiagnosticInfoResourceLimit(Fn, "stack size", StackSize, Severity, + DK_StackSize, StackLimit) {} uint64_t getStackSize() const { return getResourceSize(); } uint64_t getStackLimit() const { return getResourceLimit(); } @@ -244,7 +238,7 @@ public: /// \p The module that is concerned by this debug metadata version diagnostic. /// \p The actual metadata version. DiagnosticInfoDebugMetadataVersion(const Module &M, unsigned MetadataVersion, - DiagnosticSeverity Severity = DS_Warning) + DiagnosticSeverity Severity = DS_Warning) : DiagnosticInfo(DK_DebugMetadataVersion, Severity), M(M), MetadataVersion(MetadataVersion) {} @@ -411,7 +405,7 @@ public: /// \brief Used in the streaming interface as the general argument type. It /// internally converts everything into a key-value pair. struct Argument { - StringRef Key; + std::string Key; std::string Val; // If set, the debug location corresponding to the value. DiagnosticLocation Loc; @@ -419,9 +413,15 @@ public: explicit Argument(StringRef Str = "") : Key("String"), Val(Str) {} Argument(StringRef Key, const Value *V); Argument(StringRef Key, const Type *T); + Argument(StringRef Key, StringRef S); Argument(StringRef Key, int N); + Argument(StringRef Key, long N); + Argument(StringRef Key, long long N); Argument(StringRef Key, unsigned N); + Argument(StringRef Key, unsigned long N); + Argument(StringRef Key, unsigned long long N); Argument(StringRef Key, bool B) : Key(Key), Val(B ? "true" : "false") {} + Argument(StringRef Key, DebugLoc dl); }; /// \p PassName is the name of the pass emitting this diagnostic. \p @@ -438,10 +438,10 @@ public: : DiagnosticInfoWithLocationBase(Kind, Severity, Fn, Loc), PassName(PassName), RemarkName(RemarkName) {} - DiagnosticInfoOptimizationBase &operator<<(StringRef S); - DiagnosticInfoOptimizationBase &operator<<(Argument A); - DiagnosticInfoOptimizationBase &operator<<(setIsVerbose V); - DiagnosticInfoOptimizationBase &operator<<(setExtraArgs EA); + void insert(StringRef S); + void insert(Argument A); + void insert(setIsVerbose V); + void insert(setExtraArgs EA); /// \see DiagnosticInfo::print. void print(DiagnosticPrinter &DP) const override; @@ -511,6 +511,81 @@ protected: friend struct yaml::MappingTraits<DiagnosticInfoOptimizationBase *>; }; +/// Allow the insertion operator to return the actual remark type rather than a +/// common base class. This allows returning the result of the insertion +/// directly by value, e.g. return OptimizationRemarkAnalysis(...) << "blah". +template <class RemarkT> +RemarkT & +operator<<(RemarkT &R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + StringRef>::type S) { + R.insert(S); + return R; +} + +/// Also allow r-value for the remark to allow insertion into a +/// temporarily-constructed remark. +template <class RemarkT> +RemarkT & +operator<<(RemarkT &&R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + StringRef>::type S) { + R.insert(S); + return R; +} + +template <class RemarkT> +RemarkT & +operator<<(RemarkT &R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + DiagnosticInfoOptimizationBase::Argument>::type A) { + R.insert(A); + return R; +} + +template <class RemarkT> +RemarkT & +operator<<(RemarkT &&R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + DiagnosticInfoOptimizationBase::Argument>::type A) { + R.insert(A); + return R; +} + +template <class RemarkT> +RemarkT & +operator<<(RemarkT &R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + DiagnosticInfoOptimizationBase::setIsVerbose>::type V) { + R.insert(V); + return R; +} + +template <class RemarkT> +RemarkT & +operator<<(RemarkT &&R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + DiagnosticInfoOptimizationBase::setIsVerbose>::type V) { + R.insert(V); + return R; +} + +template <class RemarkT> +RemarkT & +operator<<(RemarkT &R, + typename std::enable_if< + std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value, + DiagnosticInfoOptimizationBase::setExtraArgs>::type EA) { + R.insert(EA); + return R; +} + /// \brief Common features for diagnostics dealing with optimization remarks /// that are used by IR passes. class DiagnosticInfoIROptimization : public DiagnosticInfoOptimizationBase { @@ -604,10 +679,8 @@ public: return DI->getKind() == DK_OptimizationRemark; } - static bool isEnabled(StringRef PassName); - /// \see DiagnosticInfoOptimizationBase::isEnabled. - bool isEnabled() const override { return isEnabled(getPassName()); } + bool isEnabled() const override; private: /// This is deprecated now and only used by the function API below. @@ -623,11 +696,6 @@ private: const DiagnosticLocation &Loc, const Twine &Msg) : DiagnosticInfoIROptimization(DK_OptimizationRemark, DS_Remark, PassName, Fn, Loc, Msg) {} - - friend void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); }; /// Diagnostic information for missed-optimization remarks. @@ -652,10 +720,8 @@ public: return DI->getKind() == DK_OptimizationRemarkMissed; } - static bool isEnabled(StringRef PassName); - /// \see DiagnosticInfoOptimizationBase::isEnabled. - bool isEnabled() const override { return isEnabled(getPassName()); } + bool isEnabled() const override; private: /// This is deprecated now and only used by the function API below. @@ -671,12 +737,6 @@ private: const DiagnosticLocation &Loc, const Twine &Msg) : DiagnosticInfoIROptimization(DK_OptimizationRemarkMissed, DS_Remark, PassName, Fn, Loc, Msg) {} - - friend void emitOptimizationRemarkMissed(LLVMContext &Ctx, - const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); }; /// Diagnostic information for optimization analysis remarks. @@ -712,12 +772,8 @@ public: return DI->getKind() == DK_OptimizationRemarkAnalysis; } - static bool isEnabled(StringRef PassName); - /// \see DiagnosticInfoOptimizationBase::isEnabled. - bool isEnabled() const override { - return shouldAlwaysPrint() || isEnabled(getPassName()); - } + bool isEnabled() const override; static const char *AlwaysPrint; @@ -748,12 +804,6 @@ private: const DiagnosticLocation &Loc, const Twine &Msg) : DiagnosticInfoIROptimization(DK_OptimizationRemarkAnalysis, DS_Remark, PassName, Fn, Loc, Msg) {} - - friend void emitOptimizationRemarkAnalysis(LLVMContext &Ctx, - const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); }; /// Diagnostic information for optimization analysis remarks related to @@ -795,10 +845,6 @@ private: const Twine &Msg) : OptimizationRemarkAnalysis(DK_OptimizationRemarkAnalysisFPCommute, PassName, Fn, Loc, Msg) {} - - friend void emitOptimizationRemarkAnalysisFPCommute( - LLVMContext &Ctx, const char *PassName, const Function &Fn, - const DiagnosticLocation &Loc, const Twine &Msg); }; /// Diagnostic information for optimization analysis remarks related to @@ -839,10 +885,6 @@ private: const Twine &Msg) : OptimizationRemarkAnalysis(DK_OptimizationRemarkAnalysisAliasing, PassName, Fn, Loc, Msg) {} - - friend void emitOptimizationRemarkAnalysisAliasing( - LLVMContext &Ctx, const char *PassName, const Function &Fn, - const DiagnosticLocation &Loc, const Twine &Msg); }; /// Diagnostic information for machine IR parser. @@ -885,74 +927,6 @@ public: // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DiagnosticInfo, LLVMDiagnosticInfoRef) -/// \brief Legacy interface to emit an optimization-applied message. Use -/// (Machine)OptimizationRemarkEmitter instead. -/// -/// \p PassName is the name of the pass emitting the message. If -Rpass= is -/// given and \p PassName matches the regular expression in -Rpass, then the -/// remark will be emitted. \p Fn is the function triggering the remark, \p Loc -/// is the debug location where the diagnostic is generated. \p Msg is the -/// message string to use. -void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, - const Function &Fn, const DiagnosticLocation &Loc, - const Twine &Msg); - -/// \brief Legacy interface to emit an optimization-missed message. Use -/// (Machine)OptimizationRemarkEmitter instead. -/// -/// \p PassName is the name of the pass emitting the message. If -Rpass-missed= -/// is given and \p PassName matches the regular expression in -Rpass, then the -/// remark will be emitted. \p Fn is the function triggering the remark, \p Loc -/// is the debug location where the diagnostic is generated. \p Msg is the -/// message string to use. -void emitOptimizationRemarkMissed(LLVMContext &Ctx, const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); - -/// \brief Legacy interface to emit an optimization analysis remark message. -/// Use (Machine)OptimizationRemarkEmitter instead. -/// -/// \p PassName is the name of the pass emitting the message. If -/// -Rpass-analysis= is given and \p PassName matches the regular expression in -/// -Rpass, then the remark will be emitted. \p Fn is the function triggering -/// the remark, \p Loc is the debug location where the diagnostic is -/// generated. \p Msg is the message string to use. -void emitOptimizationRemarkAnalysis(LLVMContext &Ctx, const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); - -/// \brief Legacy interface to emit an optimization analysis remark related to -/// messages about floating-point non-commutativity. Use -/// (Machine)OptimizationRemarkEmitter instead. -/// -/// \p PassName is the name of the pass emitting the message. If -/// -Rpass-analysis= is given and \p PassName matches the regular expression in -/// -Rpass, then the remark will be emitted. \p Fn is the function triggering -/// the remark, \p Loc is the debug location where the diagnostic is -/// generated. \p Msg is the message string to use. -void emitOptimizationRemarkAnalysisFPCommute(LLVMContext &Ctx, - const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); - -/// \brief Legacy interface to emit an optimization analysis remark related to -/// messages about pointer aliasing. Use (Machine)OptimizationRemarkEmitter -/// instead. -/// -/// \p PassName is the name of the pass emitting the message. -/// If -Rpass-analysis= is given and \p PassName matches the regular expression -/// in -Rpass, then the remark will be emitted. \p Fn is the function triggering -/// the remark, \p Loc is the debug location where the diagnostic is generated. -/// \p Msg is the message string to use. -void emitOptimizationRemarkAnalysisAliasing(LLVMContext &Ctx, - const char *PassName, - const Function &Fn, - const DiagnosticLocation &Loc, - const Twine &Msg); - /// Diagnostic information for optimization failures. class DiagnosticInfoOptimizationFailure : public DiagnosticInfoIROptimization { public: @@ -1013,6 +987,12 @@ public: void print(DiagnosticPrinter &DP) const override; }; +namespace yaml { +template <> struct MappingTraits<DiagnosticInfoOptimizationBase *> { + static void mapping(IO &io, DiagnosticInfoOptimizationBase *&OptDiag); +}; +} // namespace yaml + } // end namespace llvm #endif // LLVM_IR_DIAGNOSTICINFO_H diff --git a/contrib/llvm/include/llvm/IR/Dominators.h b/contrib/llvm/include/llvm/IR/Dominators.h index 5b21a2c83e4a..6ad99e516fba 100644 --- a/contrib/llvm/include/llvm/IR/Dominators.h +++ b/contrib/llvm/include/llvm/IR/Dominators.h @@ -41,9 +41,12 @@ namespace DomTreeBuilder { using BBDomTree = DomTreeBase<BasicBlock>; using BBPostDomTree = PostDomTreeBase<BasicBlock>; -extern template void Calculate<BBDomTree, Function>(BBDomTree &DT, Function &F); -extern template void Calculate<BBPostDomTree, Function>(BBPostDomTree &DT, - Function &F); +extern template struct Update<BasicBlock *>; + +using BBUpdates = ArrayRef<Update<BasicBlock *>>; + +extern template void Calculate<BBDomTree>(BBDomTree &DT); +extern template void Calculate<BBPostDomTree>(BBPostDomTree &DT); extern template void InsertEdge<BBDomTree>(BBDomTree &DT, BasicBlock *From, BasicBlock *To); @@ -57,6 +60,9 @@ extern template void DeleteEdge<BBPostDomTree>(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); +extern template void ApplyUpdates<BBDomTree>(BBDomTree &DT, BBUpdates); +extern template void ApplyUpdates<BBPostDomTree>(BBPostDomTree &DT, BBUpdates); + extern template bool Verify<BBDomTree>(const BBDomTree &DT); extern template bool Verify<BBPostDomTree>(const BBPostDomTree &DT); } // namespace DomTreeBuilder diff --git a/contrib/llvm/include/llvm/IR/Function.h b/contrib/llvm/include/llvm/IR/Function.h index 75fccc135dae..e811ae5e215a 100644 --- a/contrib/llvm/include/llvm/IR/Function.h +++ b/contrib/llvm/include/llvm/IR/Function.h @@ -30,7 +30,6 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalObject.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/OperandTraits.h" #include "llvm/IR/SymbolTableListTraits.h" #include "llvm/IR/Value.h" @@ -44,6 +43,10 @@ namespace llvm { +namespace Intrinsic { +enum ID : unsigned; +} + class AssemblyAnnotationWriter; class Constant; class DISubprogram; @@ -125,6 +128,11 @@ public: void operator=(const Function&) = delete; ~Function(); + // This is here to help easily convert from FunctionT * (Function * or + // MachineFunction *) in BlockFrequencyInfoImpl to Function * by calling + // FunctionT->getFunction(). + const Function &getFunction() const { return *this; } + static Function *Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N = "", Module *M = nullptr) { return new Function(Ty, Linkage, N, M); @@ -414,7 +422,7 @@ public: } void setOnlyAccessesArgMemory() { addFnAttr(Attribute::ArgMemOnly); } - /// @brief Determine if the function may only access memory that is + /// @brief Determine if the function may only access memory that is /// inaccessible from the IR. bool onlyAccessesInaccessibleMemory() const { return hasFnAttribute(Attribute::InaccessibleMemOnly); @@ -482,7 +490,7 @@ public: } void setDoesNotRecurse() { addFnAttr(Attribute::NoRecurse); - } + } /// @brief True if the ABI mandates (or the user requested) that this /// function be in a unwind table. diff --git a/contrib/llvm/include/llvm/IR/GlobalValue.h b/contrib/llvm/include/llvm/IR/GlobalValue.h index d65d43cc5957..1793de7887fc 100644 --- a/contrib/llvm/include/llvm/IR/GlobalValue.h +++ b/contrib/llvm/include/llvm/IR/GlobalValue.h @@ -80,13 +80,14 @@ protected: ValueType(Ty), Linkage(Linkage), Visibility(DefaultVisibility), UnnamedAddrVal(unsigned(UnnamedAddr::None)), DllStorageClass(DefaultStorageClass), ThreadLocal(NotThreadLocal), - HasLLVMReservedName(false), IntID((Intrinsic::ID)0U), Parent(nullptr) { + HasLLVMReservedName(false), IsDSOLocal(false), + IntID((Intrinsic::ID)0U), Parent(nullptr) { setName(Name); } Type *ValueType; - static const unsigned GlobalValueSubClassDataBits = 18; + static const unsigned GlobalValueSubClassDataBits = 17; // All bitfields use unsigned as the underlying type so that MSVC will pack // them. @@ -103,11 +104,15 @@ protected: /// Function::intrinsicID() returns Intrinsic::not_intrinsic. unsigned HasLLVMReservedName : 1; + /// If true then there is a definition within the same linkage unit and that + /// definition cannot be runtime preempted. + unsigned IsDSOLocal : 1; + private: friend class Constant; // Give subclasses access to what otherwise would be wasted padding. - // (18 + 4 + 2 + 2 + 2 + 3 + 1) == 32. + // (17 + 4 + 2 + 2 + 2 + 3 + 1 + 1) == 32. unsigned SubClassData : GlobalValueSubClassDataBits; void destroyConstantImpl(); @@ -261,6 +266,12 @@ public: Type *getValueType() const { return ValueType; } + void setDSOLocal(bool Local) { IsDSOLocal = Local; } + + bool isDSOLocal() const { + return IsDSOLocal; + } + static LinkageTypes getLinkOnceLinkage(bool ODR) { return ODR ? LinkOnceODRLinkage : LinkOnceAnyLinkage; } diff --git a/contrib/llvm/include/llvm/IR/IRBuilder.h b/contrib/llvm/include/llvm/IR/IRBuilder.h index 5344a93efb33..e687ca689d46 100644 --- a/contrib/llvm/include/llvm/IR/IRBuilder.h +++ b/contrib/llvm/include/llvm/IR/IRBuilder.h @@ -438,22 +438,26 @@ public: /// \brief Create and insert an element unordered-atomic memcpy between the /// specified pointers. /// + /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively. + /// /// If the pointers aren't i8*, they will be converted. If a TBAA tag is /// specified, it will be added to the instruction. Likewise with alias.scope /// and noalias tags. CallInst *CreateElementUnorderedAtomicMemCpy( - Value *Dst, Value *Src, uint64_t Size, uint32_t ElementSize, - MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, - MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) { + Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, + uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr) { return CreateElementUnorderedAtomicMemCpy( - Dst, Src, getInt64(Size), ElementSize, TBAATag, TBAAStructTag, ScopeTag, - NoAliasTag); + Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag, + TBAAStructTag, ScopeTag, NoAliasTag); } CallInst *CreateElementUnorderedAtomicMemCpy( - Value *Dst, Value *Src, Value *Size, uint32_t ElementSize, - MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, - MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr); + Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, + uint32_t ElementSize, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr); /// \brief Create and insert a memmove between the specified /// pointers. @@ -1806,26 +1810,28 @@ public: /// \brief Create an invariant.group.barrier intrinsic call, that stops /// optimizer to propagate equality using invariant.group metadata. - /// If Ptr type is different from i8*, it's casted to i8* before call - /// and casted back to Ptr type after call. + /// If Ptr type is different from pointer to i8, it's casted to pointer to i8 + /// in the same address space before call and casted back to Ptr type after + /// call. Value *CreateInvariantGroupBarrier(Value *Ptr) { + assert(isa<PointerType>(Ptr->getType()) && + "invariant.group.barrier only applies to pointers."); + auto *PtrType = Ptr->getType(); + auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); + if (PtrType != Int8PtrTy) + Ptr = CreateBitCast(Ptr, Int8PtrTy); Module *M = BB->getParent()->getParent(); - Function *FnInvariantGroupBarrier = Intrinsic::getDeclaration(M, - Intrinsic::invariant_group_barrier); - - Type *ArgumentAndReturnType = FnInvariantGroupBarrier->getReturnType(); - assert(ArgumentAndReturnType == - FnInvariantGroupBarrier->getFunctionType()->getParamType(0) && - "InvariantGroupBarrier should take and return the same type"); - Type *PtrType = Ptr->getType(); + Function *FnInvariantGroupBarrier = Intrinsic::getDeclaration( + M, Intrinsic::invariant_group_barrier, {Int8PtrTy}); - bool PtrTypeConversionNeeded = PtrType != ArgumentAndReturnType; - if (PtrTypeConversionNeeded) - Ptr = CreateBitCast(Ptr, ArgumentAndReturnType); + assert(FnInvariantGroupBarrier->getReturnType() == Int8PtrTy && + FnInvariantGroupBarrier->getFunctionType()->getParamType(0) == + Int8PtrTy && + "InvariantGroupBarrier should take and return the same type"); CallInst *Fn = CreateCall(FnInvariantGroupBarrier, {Ptr}); - if (PtrTypeConversionNeeded) + if (PtrType != Int8PtrTy) return CreateBitCast(Fn, PtrType); return Fn; } diff --git a/contrib/llvm/include/llvm/IR/InlineAsm.h b/contrib/llvm/include/llvm/IR/InlineAsm.h index 59874b05b0ce..1519a45d59e9 100644 --- a/contrib/llvm/include/llvm/IR/InlineAsm.h +++ b/contrib/llvm/include/llvm/IR/InlineAsm.h @@ -101,7 +101,7 @@ public: /// input constraint is required to match it (e.g. "0"). The value is the /// constraint number that matches this one (for example, if this is /// constraint #0 and constraint #4 has the value "0", this will be 4). - signed char MatchingInput = -1; + int MatchingInput = -1; /// Code - The constraint code, either the register name (in braces) or the /// constraint letter/number. @@ -128,7 +128,7 @@ public: /// input constraint is required to match it (e.g. "0"). The value is the /// constraint number that matches this one (for example, if this is /// constraint #0 and constraint #4 has the value "0", this will be 4). - signed char MatchingInput = -1; + int MatchingInput = -1; /// hasMatchingInput - Return true if this is an output constraint that has /// a matching input constraint. diff --git a/contrib/llvm/include/llvm/IR/InstrTypes.h b/contrib/llvm/include/llvm/IR/InstrTypes.h index d749077fd34a..871f702f95f2 100644 --- a/contrib/llvm/include/llvm/IR/InstrTypes.h +++ b/contrib/llvm/include/llvm/IR/InstrTypes.h @@ -775,28 +775,21 @@ public: /// A no-op cast is one that can be effected without changing any bits. /// It implies that the source and destination types are the same size. The - /// IntPtrTy argument is used to make accurate determinations for casts + /// DataLayout argument is to determine the pointer size when examining casts /// involving Integer and Pointer types. They are no-op casts if the integer /// is the same size as the pointer. However, pointer size varies with - /// platform. Generally, the result of DataLayout::getIntPtrType() should be - /// passed in. If that's not available, use Type::Int64Ty, which will make - /// the isNoopCast call conservative. + /// platform. /// @brief Determine if the described cast is a no-op cast. static bool isNoopCast( - Instruction::CastOps Opcode, ///< Opcode of cast - Type *SrcTy, ///< SrcTy of cast - Type *DstTy, ///< DstTy of cast - Type *IntPtrTy ///< Integer type corresponding to Ptr types + Instruction::CastOps Opcode, ///< Opcode of cast + Type *SrcTy, ///< SrcTy of cast + Type *DstTy, ///< DstTy of cast + const DataLayout &DL ///< DataLayout to get the Int Ptr type from. ); /// @brief Determine if this cast is a no-op cast. - bool isNoopCast( - Type *IntPtrTy ///< Integer type corresponding to pointer - ) const; - - /// @brief Determine if this cast is a no-op cast. /// - /// \param DL is the DataLayout to get the Int Ptr type from. + /// \param DL is the DataLayout to determine pointer size. bool isNoopCast(const DataLayout &DL) const; /// Determine how a pair of casts can be eliminated, if they can be at all. @@ -1487,6 +1480,12 @@ protected: default: return false; + case Attribute::InaccessibleMemOrArgMemOnly: + return hasReadingOperandBundles(); + + case Attribute::InaccessibleMemOnly: + return hasReadingOperandBundles(); + case Attribute::ArgMemOnly: return hasReadingOperandBundles(); diff --git a/contrib/llvm/include/llvm/IR/Instruction.h b/contrib/llvm/include/llvm/IR/Instruction.h index 8dc02111b866..6af9cbfae5de 100644 --- a/contrib/llvm/include/llvm/IR/Instruction.h +++ b/contrib/llvm/include/llvm/IR/Instruction.h @@ -34,6 +34,7 @@ namespace llvm { class BasicBlock; class FastMathFlags; class MDNode; +class Module; struct AAMDNodes; template <> struct ilist_alloc_traits<Instruction> { @@ -113,6 +114,10 @@ public: /// \pre I is a valid iterator into BB. void moveBefore(BasicBlock &BB, SymbolTableList<Instruction>::iterator I); + /// Unlink this instruction from its current basic block and insert it into + /// the basic block that MovePos lives in, right after MovePos. + void moveAfter(Instruction *MovePos); + //===--------------------------------------------------------------------===// // Subclass classification. //===--------------------------------------------------------------------===// @@ -304,10 +309,15 @@ public: /// Determine whether the exact flag is set. bool isExact() const; - /// Set or clear the unsafe-algebra flag on this instruction, which must be an + /// Set or clear all fast-math-flags on this instruction, which must be an /// operator which supports this flag. See LangRef.html for the meaning of /// this flag. - void setHasUnsafeAlgebra(bool B); + void setFast(bool B); + + /// Set or clear the reassociation flag on this instruction, which must be + /// an operator which supports this flag. See LangRef.html for the meaning of + /// this flag. + void setHasAllowReassoc(bool B); /// Set or clear the no-nans flag on this instruction, which must be an /// operator which supports this flag. See LangRef.html for the meaning of @@ -329,6 +339,11 @@ public: /// this flag. void setHasAllowReciprocal(bool B); + /// Set or clear the approximate-math-functions flag on this instruction, + /// which must be an operator which supports this flag. See LangRef.html for + /// the meaning of this flag. + void setHasApproxFunc(bool B); + /// Convenience function for setting multiple fast-math flags on this /// instruction, which must be an operator which supports these flags. See /// LangRef.html for the meaning of these flags. @@ -339,8 +354,11 @@ public: /// LangRef.html for the meaning of these flags. void copyFastMathFlags(FastMathFlags FMF); - /// Determine whether the unsafe-algebra flag is set. - bool hasUnsafeAlgebra() const; + /// Determine whether all fast-math-flags are set. + bool isFast() const; + + /// Determine whether the allow-reassociation flag is set. + bool hasAllowReassoc() const; /// Determine whether the no-NaNs flag is set. bool hasNoNaNs() const; @@ -357,6 +375,9 @@ public: /// Determine whether the allow-contract flag is set. bool hasAllowContract() const; + /// Determine whether the approximate-math-functions flag is set. + bool hasApproxFunc() const; + /// Convenience function for getting all the fast-math flags, which must be an /// operator which supports these flags. See LangRef.html for the meaning of /// these flags. @@ -373,6 +394,21 @@ public: /// V and this instruction. void andIRFlags(const Value *V); + /// Merge 2 debug locations and apply it to the Instruction. If the + /// instruction is a CallIns, we need to traverse the inline chain to find + /// the common scope. This is not efficient for N-way merging as each time + /// you merge 2 iterations, you need to rebuild the hashmap to find the + /// common scope. However, we still choose this API because: + /// 1) Simplicity: it takes 2 locations instead of a list of locations. + /// 2) In worst case, it increases the complexity from O(N*I) to + /// O(2*N*I), where N is # of Instructions to merge, and I is the + /// maximum level of inline stack. So it is still linear. + /// 3) Merging of call instructions should be extremely rare in real + /// applications, thus the N-way merging should be in code path. + /// The DebugLoc attached to this instruction will be overwritten by the + /// merged DebugLoc. + void applyMergedLocation(const DILocation *LocA, const DILocation *LocB); + private: /// Return true if we have an entry in the on-the-side metadata hash. bool hasMetadataHashEntry() const { diff --git a/contrib/llvm/include/llvm/IR/Instructions.h b/contrib/llvm/include/llvm/IR/Instructions.h index 60ae98869e55..c1122d137f24 100644 --- a/contrib/llvm/include/llvm/IR/Instructions.h +++ b/contrib/llvm/include/llvm/IR/Instructions.h @@ -1757,6 +1757,9 @@ public: !hasFnAttrImpl(Attribute::Builtin); } + /// Determine if the call requires strict floating point semantics. + bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); } + /// Return true if the call should not be inlined. bool isNoInline() const { return hasFnAttr(Attribute::NoInline); } void setIsNoInline() { @@ -1804,6 +1807,24 @@ public: addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly); } + /// @brief Determine if the function may only access memory that is + /// inaccessible from the IR. + bool onlyAccessesInaccessibleMemory() const { + return hasFnAttr(Attribute::InaccessibleMemOnly); + } + void setOnlyAccessesInaccessibleMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly); + } + + /// @brief Determine if the function may only access memory that is + /// either inaccessible from the IR or pointed to by its arguments. + bool onlyAccessesInaccessibleMemOrArgMem() const { + return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + } + void setOnlyAccessesInaccessibleMemOrArgMem() { + addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly); + } + /// Determine if the call cannot return. bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); } void setDoesNotReturn() { @@ -3844,6 +3865,9 @@ public: !hasFnAttrImpl(Attribute::Builtin); } + /// Determine if the call requires strict floating point semantics. + bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); } + /// Return true if the call should not be inlined. bool isNoInline() const { return hasFnAttr(Attribute::NoInline); } void setIsNoInline() { @@ -3883,6 +3907,24 @@ public: addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly); } + /// @brief Determine if the function may only access memory that is + /// inaccessible from the IR. + bool onlyAccessesInaccessibleMemory() const { + return hasFnAttr(Attribute::InaccessibleMemOnly); + } + void setOnlyAccessesInaccessibleMemory() { + addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly); + } + + /// @brief Determine if the function may only access memory that is + /// either inaccessible from the IR or pointed to by its arguments. + bool onlyAccessesInaccessibleMemOrArgMem() const { + return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + } + void setOnlyAccessesInaccessibleMemOrArgMem() { + addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly); + } + /// Determine if the call cannot return. bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); } void setDoesNotReturn() { @@ -4195,11 +4237,10 @@ private: } public: - using DerefFnTy = std::pointer_to_unary_function<Value *, BasicBlock *>; + using DerefFnTy = BasicBlock *(*)(Value *); using handler_iterator = mapped_iterator<op_iterator, DerefFnTy>; using handler_range = iterator_range<handler_iterator>; - using ConstDerefFnTy = - std::pointer_to_unary_function<const Value *, const BasicBlock *>; + using ConstDerefFnTy = const BasicBlock *(*)(const Value *); using const_handler_iterator = mapped_iterator<const_op_iterator, ConstDerefFnTy>; using const_handler_range = iterator_range<const_handler_iterator>; diff --git a/contrib/llvm/include/llvm/IR/IntrinsicInst.h b/contrib/llvm/include/llvm/IR/IntrinsicInst.h index f55d17ec72c8..2ca0a24cbae1 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicInst.h +++ b/contrib/llvm/include/llvm/IR/IntrinsicInst.h @@ -71,11 +71,35 @@ namespace llvm { /// variable's value or its address. Value *getVariableLocation(bool AllowNullOp = true) const; - // Methods for support type inquiry through isa, cast, and dyn_cast: + /// Does this describe the address of a local variable. True for dbg.addr + /// and dbg.declare, but not dbg.value, which describes its value. + bool isAddressOfVariable() const { + return getIntrinsicID() != Intrinsic::dbg_value; + } + + DILocalVariable *getVariable() const { + return cast<DILocalVariable>(getRawVariable()); + } + + DIExpression *getExpression() const { + return cast<DIExpression>(getRawExpression()); + } + + Metadata *getRawVariable() const { + return cast<MetadataAsValue>(getArgOperand(1))->getMetadata(); + } + + Metadata *getRawExpression() const { + return cast<MetadataAsValue>(getArgOperand(2))->getMetadata(); + } + + /// \name Casting methods + /// @{ static bool classof(const IntrinsicInst *I) { switch (I->getIntrinsicID()) { case Intrinsic::dbg_declare: case Intrinsic::dbg_value: + case Intrinsic::dbg_addr: return true; default: return false; } @@ -83,6 +107,7 @@ namespace llvm { static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + /// @} }; /// This represents the llvm.dbg.declare instruction. @@ -90,25 +115,26 @@ namespace llvm { public: Value *getAddress() const { return getVariableLocation(); } - DILocalVariable *getVariable() const { - return cast<DILocalVariable>(getRawVariable()); - } - - DIExpression *getExpression() const { - return cast<DIExpression>(getRawExpression()); + /// \name Casting methods + /// @{ + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::dbg_declare; } - - Metadata *getRawVariable() const { - return cast<MetadataAsValue>(getArgOperand(1))->getMetadata(); + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + /// @} + }; - Metadata *getRawExpression() const { - return cast<MetadataAsValue>(getArgOperand(2))->getMetadata(); - } + /// This represents the llvm.dbg.addr instruction. + class DbgAddrIntrinsic : public DbgInfoIntrinsic { + public: + Value *getAddress() const { return getVariableLocation(); } - // Methods for support type inquiry through isa, cast, and dyn_cast: + /// \name Casting methods + /// @{ static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::dbg_declare; + return I->getIntrinsicID() == Intrinsic::dbg_addr; } static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); @@ -122,34 +148,15 @@ namespace llvm { return getVariableLocation(/* AllowNullOp = */ false); } - uint64_t getOffset() const { - return cast<ConstantInt>( - const_cast<Value*>(getArgOperand(1)))->getZExtValue(); - } - - DILocalVariable *getVariable() const { - return cast<DILocalVariable>(getRawVariable()); - } - - DIExpression *getExpression() const { - return cast<DIExpression>(getRawExpression()); - } - - Metadata *getRawVariable() const { - return cast<MetadataAsValue>(getArgOperand(2))->getMetadata(); - } - - Metadata *getRawExpression() const { - return cast<MetadataAsValue>(getArgOperand(3))->getMetadata(); - } - - // Methods for support type inquiry through isa, cast, and dyn_cast: + /// \name Casting methods + /// @{ static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::dbg_value; } static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + /// @} }; /// This is the common base class for constrained floating point intrinsics. @@ -172,6 +179,7 @@ namespace llvm { }; bool isUnaryOp() const; + bool isTernaryOp() const; RoundingMode getRoundingMode() const; ExceptionBehavior getExceptionBehavior() const; @@ -183,6 +191,7 @@ namespace llvm { case Intrinsic::experimental_constrained_fmul: case Intrinsic::experimental_constrained_fdiv: case Intrinsic::experimental_constrained_frem: + case Intrinsic::experimental_constrained_fma: case Intrinsic::experimental_constrained_sqrt: case Intrinsic::experimental_constrained_pow: case Intrinsic::experimental_constrained_powi: @@ -204,12 +213,13 @@ namespace llvm { } }; - /// This class represents atomic memcpy intrinsic - /// TODO: Integrate this class into MemIntrinsic hierarchy; for now this is - /// C&P of all methods from that hierarchy - class ElementUnorderedAtomicMemCpyInst : public IntrinsicInst { + /// Common base class for all memory intrinsics. Simply provides + /// common methods. + /// Written as CRTP to avoid a common base class amongst the + /// three atomicity hierarchies. + template <typename Derived> class MemIntrinsicBase : public IntrinsicInst { private: - enum { ARG_DEST = 0, ARG_SOURCE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 }; + enum { ARG_DEST = 0, ARG_LENGTH = 2 }; public: Value *getRawDest() const { @@ -218,51 +228,21 @@ namespace llvm { const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); } Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); } - /// Return the arguments to the instruction. - Value *getRawSource() const { - return const_cast<Value *>(getArgOperand(ARG_SOURCE)); - } - const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); } - Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); } - Value *getLength() const { return const_cast<Value *>(getArgOperand(ARG_LENGTH)); } const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); } Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); } - bool isVolatile() const { return false; } - - Value *getRawElementSizeInBytes() const { - return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE)); - } - - ConstantInt *getElementSizeInBytesCst() const { - return cast<ConstantInt>(getRawElementSizeInBytes()); - } - - uint32_t getElementSizeInBytes() const { - return getElementSizeInBytesCst()->getZExtValue(); - } - /// This is just like getRawDest, but it strips off any cast - /// instructions that feed it, giving the original input. The returned - /// value is guaranteed to be a pointer. + /// instructions (including addrspacecast) that feed it, giving the + /// original input. The returned value is guaranteed to be a pointer. Value *getDest() const { return getRawDest()->stripPointerCasts(); } - /// This is just like getRawSource, but it strips off any cast - /// instructions that feed it, giving the original input. The returned - /// value is guaranteed to be a pointer. - Value *getSource() const { return getRawSource()->stripPointerCasts(); } - unsigned getDestAddressSpace() const { return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); } - unsigned getSourceAddressSpace() const { - return cast<PointerType>(getRawSource()->getType())->getAddressSpace(); - } - /// Set the specified arguments of the instruction. void setDest(Value *Ptr) { assert(getRawDest()->getType() == Ptr->getType() && @@ -270,58 +250,20 @@ namespace llvm { setArgOperand(ARG_DEST, Ptr); } - void setSource(Value *Ptr) { - assert(getRawSource()->getType() == Ptr->getType() && - "setSource called with pointer of wrong type!"); - setArgOperand(ARG_SOURCE, Ptr); - } - void setLength(Value *L) { assert(getLength()->getType() == L->getType() && "setLength called with value of wrong type!"); setArgOperand(ARG_LENGTH, L); } - - void setElementSizeInBytes(Constant *V) { - assert(V->getType() == Type::getInt8Ty(getContext()) && - "setElementSizeInBytes called with value of wrong type!"); - setArgOperand(ARG_ELEMENTSIZE, V); - } - - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::memcpy_element_unordered_atomic; - } - static bool classof(const Value *V) { - return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); - } }; - class ElementUnorderedAtomicMemMoveInst : public IntrinsicInst { + // The common base class for the atomic memset/memmove/memcpy intrinsics + // i.e. llvm.element.unordered.atomic.memset/memcpy/memmove + class AtomicMemIntrinsic : public MemIntrinsicBase<AtomicMemIntrinsic> { private: - enum { ARG_DEST = 0, ARG_SOURCE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 }; + enum { ARG_ELEMENTSIZE = 3 }; public: - Value *getRawDest() const { - return const_cast<Value *>(getArgOperand(ARG_DEST)); - } - const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); } - Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); } - - /// Return the arguments to the instruction. - Value *getRawSource() const { - return const_cast<Value *>(getArgOperand(ARG_SOURCE)); - } - const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); } - Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); } - - Value *getLength() const { - return const_cast<Value *>(getArgOperand(ARG_LENGTH)); - } - const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); } - Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); } - - bool isVolatile() const { return false; } - Value *getRawElementSizeInBytes() const { return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE)); } @@ -334,150 +276,129 @@ namespace llvm { return getElementSizeInBytesCst()->getZExtValue(); } - /// This is just like getRawDest, but it strips off any cast - /// instructions that feed it, giving the original input. The returned - /// value is guaranteed to be a pointer. - Value *getDest() const { return getRawDest()->stripPointerCasts(); } - - /// This is just like getRawSource, but it strips off any cast - /// instructions that feed it, giving the original input. The returned - /// value is guaranteed to be a pointer. - Value *getSource() const { return getRawSource()->stripPointerCasts(); } - - unsigned getDestAddressSpace() const { - return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); - } - - unsigned getSourceAddressSpace() const { - return cast<PointerType>(getRawSource()->getType())->getAddressSpace(); - } - - /// Set the specified arguments of the instruction. - void setDest(Value *Ptr) { - assert(getRawDest()->getType() == Ptr->getType() && - "setDest called with pointer of wrong type!"); - setArgOperand(ARG_DEST, Ptr); - } - - void setSource(Value *Ptr) { - assert(getRawSource()->getType() == Ptr->getType() && - "setSource called with pointer of wrong type!"); - setArgOperand(ARG_SOURCE, Ptr); - } - - void setLength(Value *L) { - assert(getLength()->getType() == L->getType() && - "setLength called with value of wrong type!"); - setArgOperand(ARG_LENGTH, L); - } - void setElementSizeInBytes(Constant *V) { assert(V->getType() == Type::getInt8Ty(getContext()) && "setElementSizeInBytes called with value of wrong type!"); setArgOperand(ARG_ELEMENTSIZE, V); } - static inline bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::memmove_element_unordered_atomic; + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcpy_element_unordered_atomic: + case Intrinsic::memmove_element_unordered_atomic: + case Intrinsic::memset_element_unordered_atomic: + return true; + default: + return false; + } } - static inline bool classof(const Value *V) { + static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } }; /// This class represents atomic memset intrinsic - /// TODO: Integrate this class into MemIntrinsic hierarchy; for now this is - /// C&P of all methods from that hierarchy - class ElementUnorderedAtomicMemSetInst : public IntrinsicInst { + // i.e. llvm.element.unordered.atomic.memset + class AtomicMemSetInst : public AtomicMemIntrinsic { private: - enum { ARG_DEST = 0, ARG_VALUE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 }; + enum { ARG_VALUE = 1 }; public: - Value *getRawDest() const { - return const_cast<Value *>(getArgOperand(ARG_DEST)); + Value *getValue() const { + return const_cast<Value *>(getArgOperand(ARG_VALUE)); } - const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); } - Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); } - - Value *getValue() const { return const_cast<Value*>(getArgOperand(ARG_VALUE)); } const Use &getValueUse() const { return getArgOperandUse(ARG_VALUE); } Use &getValueUse() { return getArgOperandUse(ARG_VALUE); } - Value *getLength() const { - return const_cast<Value *>(getArgOperand(ARG_LENGTH)); + void setValue(Value *Val) { + assert(getValue()->getType() == Val->getType() && + "setValue called with value of wrong type!"); + setArgOperand(ARG_VALUE, Val); } - const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); } - Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); } - - bool isVolatile() const { return false; } - Value *getRawElementSizeInBytes() const { - return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE)); + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memset_element_unordered_atomic; } - - ConstantInt *getElementSizeInBytesCst() const { - return cast<ConstantInt>(getRawElementSizeInBytes()); + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + }; - uint32_t getElementSizeInBytes() const { - return getElementSizeInBytesCst()->getZExtValue(); + // This class wraps the atomic memcpy/memmove intrinsics + // i.e. llvm.element.unordered.atomic.memcpy/memmove + class AtomicMemTransferInst : public AtomicMemIntrinsic { + private: + enum { ARG_SOURCE = 1 }; + + public: + /// Return the arguments to the instruction. + Value *getRawSource() const { + return const_cast<Value *>(getArgOperand(ARG_SOURCE)); } + const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); } + Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); } - /// This is just like getRawDest, but it strips off any cast + /// This is just like getRawSource, but it strips off any cast /// instructions that feed it, giving the original input. The returned /// value is guaranteed to be a pointer. - Value *getDest() const { return getRawDest()->stripPointerCasts(); } + Value *getSource() const { return getRawSource()->stripPointerCasts(); } - unsigned getDestAddressSpace() const { - return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); + unsigned getSourceAddressSpace() const { + return cast<PointerType>(getRawSource()->getType())->getAddressSpace(); } - /// Set the specified arguments of the instruction. - void setDest(Value *Ptr) { - assert(getRawDest()->getType() == Ptr->getType() && - "setDest called with pointer of wrong type!"); - setArgOperand(ARG_DEST, Ptr); + void setSource(Value *Ptr) { + assert(getRawSource()->getType() == Ptr->getType() && + "setSource called with pointer of wrong type!"); + setArgOperand(ARG_SOURCE, Ptr); } - void setValue(Value *Val) { - assert(getValue()->getType() == Val->getType() && - "setValue called with value of wrong type!"); - setArgOperand(ARG_VALUE, Val); + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcpy_element_unordered_atomic: + case Intrinsic::memmove_element_unordered_atomic: + return true; + default: + return false; + } } - - void setLength(Value *L) { - assert(getLength()->getType() == L->getType() && - "setLength called with value of wrong type!"); - setArgOperand(ARG_LENGTH, L); + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + }; - void setElementSizeInBytes(Constant *V) { - assert(V->getType() == Type::getInt8Ty(getContext()) && - "setElementSizeInBytes called with value of wrong type!"); - setArgOperand(ARG_ELEMENTSIZE, V); + /// This class represents the atomic memcpy intrinsic + /// i.e. llvm.element.unordered.atomic.memcpy + class AtomicMemCpyInst : public AtomicMemTransferInst { + public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memcpy_element_unordered_atomic; + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } + }; - static inline bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::memset_element_unordered_atomic; + /// This class represents the atomic memmove intrinsic + /// i.e. llvm.element.unordered.atomic.memmove + class AtomicMemMoveInst : public AtomicMemTransferInst { + public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::memmove_element_unordered_atomic; } - static inline bool classof(const Value *V) { + static bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } }; /// This is the common base class for memset/memcpy/memmove. - class MemIntrinsic : public IntrinsicInst { - public: - Value *getRawDest() const { return const_cast<Value*>(getArgOperand(0)); } - const Use &getRawDestUse() const { return getArgOperandUse(0); } - Use &getRawDestUse() { return getArgOperandUse(0); } - - Value *getLength() const { return const_cast<Value*>(getArgOperand(2)); } - const Use &getLengthUse() const { return getArgOperandUse(2); } - Use &getLengthUse() { return getArgOperandUse(2); } + class MemIntrinsic : public MemIntrinsicBase<MemIntrinsic> { + private: + enum { ARG_ALIGN = 3, ARG_VOLATILE = 4 }; + public: ConstantInt *getAlignmentCst() const { - return cast<ConstantInt>(const_cast<Value*>(getArgOperand(3))); + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(ARG_ALIGN))); } unsigned getAlignment() const { @@ -485,45 +406,20 @@ namespace llvm { } ConstantInt *getVolatileCst() const { - return cast<ConstantInt>(const_cast<Value*>(getArgOperand(4))); + return cast<ConstantInt>( + const_cast<Value *>(getArgOperand(ARG_VOLATILE))); } bool isVolatile() const { return !getVolatileCst()->isZero(); } - unsigned getDestAddressSpace() const { - return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); - } - - /// This is just like getRawDest, but it strips off any cast - /// instructions that feed it, giving the original input. The returned - /// value is guaranteed to be a pointer. - Value *getDest() const { return getRawDest()->stripPointerCasts(); } - - /// Set the specified arguments of the instruction. - void setDest(Value *Ptr) { - assert(getRawDest()->getType() == Ptr->getType() && - "setDest called with pointer of wrong type!"); - setArgOperand(0, Ptr); - } - - void setLength(Value *L) { - assert(getLength()->getType() == L->getType() && - "setLength called with value of wrong type!"); - setArgOperand(2, L); - } - - void setAlignment(Constant* A) { - setArgOperand(3, A); - } + void setAlignment(Constant *A) { setArgOperand(ARG_ALIGN, A); } - void setVolatile(Constant* V) { - setArgOperand(4, V); - } + void setVolatile(Constant *V) { setArgOperand(ARG_VOLATILE, V); } Type *getAlignmentType() const { - return getArgOperand(3)->getType(); + return getArgOperand(ARG_ALIGN)->getType(); } // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -621,6 +517,155 @@ namespace llvm { } }; + // The common base class for any memset/memmove/memcpy intrinsics; + // whether they be atomic or non-atomic. + // i.e. llvm.element.unordered.atomic.memset/memcpy/memmove + // and llvm.memset/memcpy/memmove + class AnyMemIntrinsic : public MemIntrinsicBase<AnyMemIntrinsic> { + public: + bool isVolatile() const { + // Only the non-atomic intrinsics can be volatile + if (auto *MI = dyn_cast<MemIntrinsic>(this)) + return MI->isVolatile(); + return false; + } + + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memmove: + case Intrinsic::memset: + case Intrinsic::memcpy_element_unordered_atomic: + case Intrinsic::memmove_element_unordered_atomic: + case Intrinsic::memset_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; + + /// This class represents any memset intrinsic + // i.e. llvm.element.unordered.atomic.memset + // and llvm.memset + class AnyMemSetInst : public AnyMemIntrinsic { + private: + enum { ARG_VALUE = 1 }; + + public: + Value *getValue() const { + return const_cast<Value *>(getArgOperand(ARG_VALUE)); + } + const Use &getValueUse() const { return getArgOperandUse(ARG_VALUE); } + Use &getValueUse() { return getArgOperandUse(ARG_VALUE); } + + void setValue(Value *Val) { + assert(getValue()->getType() == Val->getType() && + "setValue called with value of wrong type!"); + setArgOperand(ARG_VALUE, Val); + } + + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memset: + case Intrinsic::memset_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; + + // This class wraps any memcpy/memmove intrinsics + // i.e. llvm.element.unordered.atomic.memcpy/memmove + // and llvm.memcpy/memmove + class AnyMemTransferInst : public AnyMemIntrinsic { + private: + enum { ARG_SOURCE = 1 }; + + public: + /// Return the arguments to the instruction. + Value *getRawSource() const { + return const_cast<Value *>(getArgOperand(ARG_SOURCE)); + } + const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); } + Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); } + + /// This is just like getRawSource, but it strips off any cast + /// instructions that feed it, giving the original input. The returned + /// value is guaranteed to be a pointer. + Value *getSource() const { return getRawSource()->stripPointerCasts(); } + + unsigned getSourceAddressSpace() const { + return cast<PointerType>(getRawSource()->getType())->getAddressSpace(); + } + + void setSource(Value *Ptr) { + assert(getRawSource()->getType() == Ptr->getType() && + "setSource called with pointer of wrong type!"); + setArgOperand(ARG_SOURCE, Ptr); + } + + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memmove: + case Intrinsic::memcpy_element_unordered_atomic: + case Intrinsic::memmove_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; + + /// This class represents any memcpy intrinsic + /// i.e. llvm.element.unordered.atomic.memcpy + /// and llvm.memcpy + class AnyMemCpyInst : public AnyMemTransferInst { + public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memcpy_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; + + /// This class represents any memmove intrinsic + /// i.e. llvm.element.unordered.atomic.memmove + /// and llvm.memmove + class AnyMemMoveInst : public AnyMemTransferInst { + public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::memmove: + case Intrinsic::memmove_element_unordered_atomic: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; + /// This represents the llvm.va_start intrinsic. class VAStartInst : public IntrinsicInst { public: diff --git a/contrib/llvm/include/llvm/IR/Intrinsics.td b/contrib/llvm/include/llvm/IR/Intrinsics.td index 14c88e519435..07de0568cab0 100644 --- a/contrib/llvm/include/llvm/IR/Intrinsics.td +++ b/contrib/llvm/include/llvm/IR/Intrinsics.td @@ -490,6 +490,13 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_fma : Intrinsic<[ llvm_anyfloat_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_metadata_ty, + llvm_metadata_ty ]>; + // These intrinsics are sensitive to the rounding mode so we need constrained // versions of each of them. When strict rounding and exception control are // not required the non-constrained versions of these intrinsics should be @@ -576,10 +583,14 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in { let IntrProperties = [IntrNoMem, IntrSpeculatable] in { def int_dbg_declare : Intrinsic<[], [llvm_metadata_ty, - llvm_metadata_ty, - llvm_metadata_ty]>; + llvm_metadata_ty, + llvm_metadata_ty]>; def int_dbg_value : Intrinsic<[], - [llvm_metadata_ty, llvm_i64_ty, + [llvm_metadata_ty, + llvm_metadata_ty, + llvm_metadata_ty]>; + def int_dbg_addr : Intrinsic<[], + [llvm_metadata_ty, llvm_metadata_ty, llvm_metadata_ty]>; } @@ -634,6 +645,13 @@ def int_annotation : Intrinsic<[llvm_anyint_ty], llvm_ptr_ty, llvm_i32_ty], [], "llvm.annotation">; +// Annotates the current program point with metadata strings which are emitted +// as CodeView debug info records. This is expensive, as it disables inlining +// and is modelled as having side effects. +def int_codeview_annotation : Intrinsic<[], [llvm_metadata_ty], + [IntrInaccessibleMemOnly, IntrNoDuplicate], + "llvm.codeview.annotation">; + //===------------------------ Trampoline Intrinsics -----------------------===// // def int_init_trampoline : Intrinsic<[], @@ -693,8 +711,8 @@ def int_invariant_end : Intrinsic<[], // which is valid. // The argument also can't be marked with 'returned' attribute, because // it would remove barrier. -def int_invariant_group_barrier : Intrinsic<[llvm_ptr_ty], - [llvm_ptr_ty], +def int_invariant_group_barrier : Intrinsic<[llvm_anyptr_ty], + [LLVMMatchType<0>], [IntrReadMem, IntrArgMemOnly]>; //===------------------------ Stackmap Intrinsics -------------------------===// @@ -792,6 +810,12 @@ def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty], // NOP: calls/invokes to this intrinsic are removed by codegen def int_donothing : Intrinsic<[], [], [IntrNoMem]>; +// This instruction has no actual effect, though it is treated by the optimizer +// has having opaque side effects. This may be inserted into loops to ensure +// that they are not removed even if they turn out to be empty, for languages +// which specify that infinite loops must be preserved. +def int_sideeffect : Intrinsic<[], [], [IntrInaccessibleMemOnly]>; + // Intrisics to support half precision floating point format let IntrProperties = [IntrNoMem] in { def int_convert_to_fp16 : Intrinsic<[llvm_i16_ty], [llvm_anyfloat_ty]>; diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsAArch64.td b/contrib/llvm/include/llvm/IR/IntrinsicsAArch64.td index 2c45d148e34b..65c9aaab975d 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -40,9 +40,9 @@ def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>; //===----------------------------------------------------------------------===// // Data Barrier Instructions -def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, Intrinsic<[], [llvm_i32_ty]>; -def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, Intrinsic<[], [llvm_i32_ty]>; -def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, Intrinsic<[], [llvm_i32_ty]>; +def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>; +def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>; +def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>; } diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/contrib/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 4e0529a32d29..d7999cd33231 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -294,7 +294,7 @@ class AMDGPUAtomicIncIntrin : Intrinsic<[llvm_anyint_ty], def int_amdgcn_atomic_inc : AMDGPUAtomicIncIntrin; def int_amdgcn_atomic_dec : AMDGPUAtomicIncIntrin; -class AMDGPUImageLoad : Intrinsic < +class AMDGPUImageLoad<bit NoMem = 0> : Intrinsic < [llvm_anyfloat_ty], // vdata(VGPR) [llvm_anyint_ty, // vaddr(VGPR) llvm_anyint_ty, // rsrc(SGPR) @@ -303,11 +303,11 @@ class AMDGPUImageLoad : Intrinsic < llvm_i1_ty, // slc(imm) llvm_i1_ty, // lwe(imm) llvm_i1_ty], // da(imm) - [IntrReadMem]>; + !if(NoMem, [IntrNoMem], [IntrReadMem])>; def int_amdgcn_image_load : AMDGPUImageLoad; def int_amdgcn_image_load_mip : AMDGPUImageLoad; -def int_amdgcn_image_getresinfo : AMDGPUImageLoad; +def int_amdgcn_image_getresinfo : AMDGPUImageLoad<1>; class AMDGPUImageStore : Intrinsic < [], @@ -324,7 +324,7 @@ class AMDGPUImageStore : Intrinsic < def int_amdgcn_image_store : AMDGPUImageStore; def int_amdgcn_image_store_mip : AMDGPUImageStore; -class AMDGPUImageSample : Intrinsic < +class AMDGPUImageSample<bit NoMem = 0> : Intrinsic < [llvm_anyfloat_ty], // vdata(VGPR) [llvm_anyfloat_ty, // vaddr(VGPR) llvm_anyint_ty, // rsrc(SGPR) @@ -335,7 +335,7 @@ class AMDGPUImageSample : Intrinsic < llvm_i1_ty, // slc(imm) llvm_i1_ty, // lwe(imm) llvm_i1_ty], // da(imm) - [IntrReadMem]>; + !if(NoMem, [IntrNoMem], [IntrReadMem])>; // Basic sample def int_amdgcn_image_sample : AMDGPUImageSample; @@ -417,7 +417,7 @@ def int_amdgcn_image_gather4_c_b_o : AMDGPUImageSample; def int_amdgcn_image_gather4_c_b_cl_o : AMDGPUImageSample; def int_amdgcn_image_gather4_c_lz_o : AMDGPUImageSample; -def int_amdgcn_image_getlod : AMDGPUImageSample; +def int_amdgcn_image_getlod : AMDGPUImageSample<1>; class AMDGPUImageAtomic : Intrinsic < [llvm_i32_ty], @@ -570,7 +570,7 @@ def int_amdgcn_s_dcache_inv : def int_amdgcn_s_memtime : GCCBuiltin<"__builtin_amdgcn_s_memtime">, - Intrinsic<[llvm_i64_ty], [], []>; + Intrinsic<[llvm_i64_ty], [], [IntrReadMem]>; def int_amdgcn_s_sleep : GCCBuiltin<"__builtin_amdgcn_s_sleep">, @@ -740,6 +740,41 @@ def int_amdgcn_alignbyte : Intrinsic<[llvm_i32_ty], [IntrNoMem, IntrSpeculatable] >; + +// Copies the source value to the destination value, with the guarantee that +// the source value is computed as if the entire program were executed in WQM. +def int_amdgcn_wqm : Intrinsic<[llvm_any_ty], + [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] +>; + +// Return true if at least one thread within the pixel quad passes true into +// the function. +def int_amdgcn_wqm_vote : Intrinsic<[llvm_i1_ty], + [llvm_i1_ty], [IntrNoMem, IntrConvergent] +>; + +// If false, set EXEC=0 for the current thread until the end of program. +def int_amdgcn_kill : Intrinsic<[], [llvm_i1_ty], []>; + +// Copies the active channels of the source value to the destination value, +// with the guarantee that the source value is computed as if the entire +// program were executed in Whole Wavefront Mode, i.e. with all channels +// enabled, with a few exceptions: - Phi nodes with require WWM return an +// undefined value. +def int_amdgcn_wwm : Intrinsic<[llvm_any_ty], + [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] +>; + +// Given a value, copies it while setting all the inactive lanes to a given +// value. Note that OpenGL helper lanes are considered active, so if the +// program ever uses WQM, then the instruction and the first source will be +// computed in WQM. +def int_amdgcn_set_inactive : + Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, // value to be copied + LLVMMatchType<0>], // value for the inactive lanes to take + [IntrNoMem, IntrConvergent]>; + //===----------------------------------------------------------------------===// // CI+ Intrinsics //===----------------------------------------------------------------------===// @@ -762,6 +797,15 @@ def int_amdgcn_mov_dpp : [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty], [IntrNoMem, IntrConvergent]>; +// llvm.amdgcn.update.dpp.i32 <old> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl> +// Should be equivalent to: +// v_mov_b32 <dest> <old> +// v_mov_b32 <dest> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl> +def int_amdgcn_update_dpp : + Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i1_ty], [IntrNoMem, IntrConvergent]>; + def int_amdgcn_s_dcache_wb : GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, Intrinsic<[], [], []>; @@ -772,7 +816,7 @@ def int_amdgcn_s_dcache_wb_vol : def int_amdgcn_s_memrealtime : GCCBuiltin<"__builtin_amdgcn_s_memrealtime">, - Intrinsic<[llvm_i64_ty], [], []>; + Intrinsic<[llvm_i64_ty], [], [IntrReadMem]>; // llvm.amdgcn.ds.permute <index> <src> def int_amdgcn_ds_permute : diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsHexagon.td b/contrib/llvm/include/llvm/IR/IntrinsicsHexagon.td index 098245344725..5c96702bca76 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsHexagon.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsHexagon.td @@ -5044,7 +5044,6 @@ def int_hexagon_V6_vassignp_128B : Hexagon_v2048v2048_Intrinsic_T<"HEXAGON_V6_vassignp_128B">; - // // Hexagon_iii_Intrinsic<string GCCIntSuffix> // tag : S6_rol_i_r @@ -5583,54 +5582,6 @@ class Hexagon_v1024i_Intrinsic<string GCCIntSuffix> [IntrNoMem]>; // -// Hexagon_v512v512LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb -class Hexagon_v512v512LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// -// Hexagon_v1024v1024LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb_128B -class Hexagon_v1024v1024LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// -// Hexagon_v512v512v512LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb_acc -class Hexagon_v512v512v512LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// -// Hexagon_v1024v1024v1024LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb_acc_128B -class Hexagon_v1024v1024v1024LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// -// Hexagon_v2048v2048LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb_dv_128B -class Hexagon_v2048v2048LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// -// Hexagon_v2048v2048v2048LLii_Intrinsic<string GCCIntSuffix> -// tag : V6_vlutb_dv_acc_128B -class Hexagon_v2048v2048v2048LLii_Intrinsic<string GCCIntSuffix> - : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i64_ty,llvm_i32_ty], - [IntrNoMem]>; - -// // Hexagon_v512v512v512v512i_Intrinsic<string GCCIntSuffix> // tag : V6_vlutvvb_oracc class Hexagon_v512v512v512v512i_Intrinsic<string GCCIntSuffix> @@ -9167,54 +9118,6 @@ def int_hexagon_V6_vcombine_128B : Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vcombine_128B">; // -// BUILTIN_INFO(HEXAGON.V6_vlutb,VI_ftype_VIDISI,3) -// tag : V6_vlutb -def int_hexagon_V6_vlutb : -Hexagon_v512v512LLii_Intrinsic<"HEXAGON_V6_vlutb">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_128B,VI_ftype_VIDISI,3) -// tag : V6_vlutb_128B -def int_hexagon_V6_vlutb_128B : -Hexagon_v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_128B">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_acc,VI_ftype_VIVIDISI,4) -// tag : V6_vlutb_acc -def int_hexagon_V6_vlutb_acc : -Hexagon_v512v512v512LLii_Intrinsic<"HEXAGON_V6_vlutb_acc">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_acc_128B,VI_ftype_VIVIDISI,4) -// tag : V6_vlutb_acc_128B -def int_hexagon_V6_vlutb_acc_128B : -Hexagon_v1024v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_acc_128B">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_dv,VD_ftype_VDDISI,3) -// tag : V6_vlutb_dv -def int_hexagon_V6_vlutb_dv : -Hexagon_v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_dv">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_128B,VD_ftype_VDDISI,3) -// tag : V6_vlutb_dv_128B -def int_hexagon_V6_vlutb_dv_128B : -Hexagon_v2048v2048LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_128B">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_acc,VD_ftype_VDVDDISI,4) -// tag : V6_vlutb_dv_acc -def int_hexagon_V6_vlutb_dv_acc : -Hexagon_v1024v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_acc">; - -// -// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_acc_128B,VD_ftype_VDVDDISI,4) -// tag : V6_vlutb_dv_acc_128B -def int_hexagon_V6_vlutb_dv_acc_128B : -Hexagon_v2048v2048v2048LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_acc_128B">; - -// // BUILTIN_INFO(HEXAGON.V6_vdelta,VI_ftype_VIVI,2) // tag : V6_vdelta def int_hexagon_V6_vdelta : @@ -9349,6 +9252,30 @@ Hexagon_v2048v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvwh_oracc_128B">; // // Masked vector stores // +def int_hexagon_V6_vS32b_qpred_ai : +Hexagon_vv64ivmemv512_Intrinsic<"HEXAGON_V6_vS32b_qpred_ai">; + +def int_hexagon_V6_vS32b_nqpred_ai : +Hexagon_vv64ivmemv512_Intrinsic<"HEXAGON_V6_vS32b_nqpred_ai">; + +def int_hexagon_V6_vS32b_nt_qpred_ai : +Hexagon_vv64ivmemv512_Intrinsic<"HEXAGON_V6_vS32b_nt_qpred_ai">; + +def int_hexagon_V6_vS32b_nt_nqpred_ai : +Hexagon_vv64ivmemv512_Intrinsic<"HEXAGON_V6_vS32b_nt_nqpred_ai">; + +def int_hexagon_V6_vS32b_qpred_ai_128B : +Hexagon_vv128ivmemv1024_Intrinsic<"HEXAGON_V6_vS32b_qpred_ai_128B">; + +def int_hexagon_V6_vS32b_nqpred_ai_128B : +Hexagon_vv128ivmemv1024_Intrinsic<"HEXAGON_V6_vS32b_nqpred_ai_128B">; + +def int_hexagon_V6_vS32b_nt_qpred_ai_128B : +Hexagon_vv128ivmemv1024_Intrinsic<"HEXAGON_V6_vS32b_nt_qpred_ai_128B">; + +def int_hexagon_V6_vS32b_nt_nqpred_ai_128B : +Hexagon_vv128ivmemv1024_Intrinsic<"HEXAGON_V6_vS32b_nt_nqpred_ai_128B">; + def int_hexagon_V6_vmaskedstoreq : Hexagon_vv64ivmemv512_Intrinsic<"HEXAGON_V6_vmaskedstoreq">; @@ -9642,6 +9569,20 @@ class Hexagon_V62_v2048v2048v1024v1024i_Intrinsic<string GCCIntSuffix> [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], [IntrNoMem]>; +// Hexagon_v512v64iv512v512v64i_Intrinsic<string GCCIntSuffix> +// tag: V6_vaddcarry +class Hexagon_v512v64iv512v512v64i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty, llvm_v512i1_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v512i1_ty], + [IntrNoMem]>; + +// Hexagon_v1024v128iv1024v1024v128i_Intrinsic<string GCCIntSuffix> +// tag: V6_vaddcarry_128B +class Hexagon_v1024v128iv1024v1024v128i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty, llvm_v1024i1_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v1024i1_ty], + [IntrNoMem]>; + // // BUILTIN_INFO(HEXAGON.M6_vabsdiffb,DI_ftype_DIDI,2) @@ -10213,3 +10154,821 @@ Hexagon_V62_v1024v512v512i_Intrinsic<"HEXAGON_V6_vlutvwh_nm">; def int_hexagon_V6_vlutvwh_nm_128B : Hexagon_V62_v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvwh_nm_128B">; +// +// BUILTIN_INFO(HEXAGON.V6_vaddcarry,VI_ftype_VIVIQV,3) +// tag: V6_vaddcarry +def int_hexagon_V6_vaddcarry : +Hexagon_v512v64iv512v512v64i_Intrinsic<"HEXAGON_v6_vaddcarry">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddcarry_128B,VI_ftype_VIVIQV,3) +// tag: V6_vaddcarry_128B +def int_hexagon_V6_vaddcarry_128B : +Hexagon_v1024v128iv1024v1024v128i_Intrinsic<"HEXAGON_v6_vaddcarry_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubcarry,VI_ftype_VIVIQV,3) +// tag: V6_vsubcarry +def int_hexagon_V6_vsubcarry : +Hexagon_v512v64iv512v512v64i_Intrinsic<"HEXAGON_v6_vsubcarry">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubcarry_128B,VI_ftype_VIVIQV,3) +// tag: V6_vsubcarry_128B +def int_hexagon_V6_vsubcarry_128B : +Hexagon_v1024v128iv1024v1024v128i_Intrinsic<"HEXAGON_v6_vsubcarry_128B">; + + +/// +/// HexagonV65 intrinsics +/// + +// +// Hexagon_V65_iLLiLLi_Intrinsic<string GCCIntSuffix> +// tag : A6_vcmpbeq_notany +class Hexagon_V65_iLLiLLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i32_ty], [llvm_i64_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v512LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyub_rtt +class Hexagon_V65_v1024v512LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v2048v1024LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyub_rtt_128B +class Hexagon_V65_v2048v1024LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024v512LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyub_rtt_acc +class Hexagon_V65_v1024v1024v512LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v2048v2048v1024LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyub_rtt_acc_128B +class Hexagon_V65_v2048v2048v1024LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vasruwuhsat +class Hexagon_V65_v512v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vasruwuhsat_128B +class Hexagon_V65_v1024v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vavguw +class Hexagon_V65_v512v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vavguw_128B +class Hexagon_V65_v1024v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vabsb +class Hexagon_V65_v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vabsb_128B +class Hexagon_V65_v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpabuu +class Hexagon_V65_v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v2048v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpabuu_128B +class Hexagon_V65_v2048v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v2048v2048v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpabuu_acc_128B +class Hexagon_V65_v2048v2048v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyh_acc +class Hexagon_V65_v1024v1024v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v2048v2048v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyh_acc_128B +class Hexagon_V65_v2048v2048v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512v512LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpahhsat +class Hexagon_V65_v512v512v512LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024v1024LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpahhsat_128B +class Hexagon_V65_v1024v1024v1024LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vlut4 +class Hexagon_V65_v512v512LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v1024LLi_Intrinsic<string GCCIntSuffix> +// tag : V6_vlut4_128B +class Hexagon_V65_v1024v1024LLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyuhe +class Hexagon_V65_v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v512v64i_Intrinsic<string GCCIntSuffix> +// tag : V6_vprefixqb +class Hexagon_V65_v512v64i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v512i1_ty], + [IntrNoMem]>; + +// +// Hexagon_V65_v1024v128i_Intrinsic<string GCCIntSuffix> +// tag : V6_vprefixqb_128B +class Hexagon_V65_v1024v128i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v1024i1_ty], + [IntrNoMem]>; + +// +// BUILTIN_INFO(HEXAGON.A6_vcmpbeq_notany,QI_ftype_DIDI,2) +// tag : A6_vcmpbeq_notany +def int_hexagon_A6_vcmpbeq_notany : +Hexagon_V65_iLLiLLi_Intrinsic<"HEXAGON_A6_vcmpbeq_notany">; + +// +// BUILTIN_INFO(HEXAGON.A6_vcmpbeq_notany_128B,QI_ftype_DIDI,2) +// tag : A6_vcmpbeq_notany_128B +def int_hexagon_A6_vcmpbeq_notany_128B : +Hexagon_V65_iLLiLLi_Intrinsic<"HEXAGON_A6_vcmpbeq_notany_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_rtt,VD_ftype_VIDI,2) +// tag : V6_vrmpyub_rtt +def int_hexagon_V6_vrmpyub_rtt : +Hexagon_V65_v1024v512LLi_Intrinsic<"HEXAGON_V6_vrmpyub_rtt">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_rtt_128B,VD_ftype_VIDI,2) +// tag : V6_vrmpyub_rtt_128B +def int_hexagon_V6_vrmpyub_rtt_128B : +Hexagon_V65_v2048v1024LLi_Intrinsic<"HEXAGON_V6_vrmpyub_rtt_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_rtt_acc,VD_ftype_VDVIDI,3) +// tag : V6_vrmpyub_rtt_acc +def int_hexagon_V6_vrmpyub_rtt_acc : +Hexagon_V65_v1024v1024v512LLi_Intrinsic<"HEXAGON_V6_vrmpyub_rtt_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_rtt_acc_128B,VD_ftype_VDVIDI,3) +// tag : V6_vrmpyub_rtt_acc_128B +def int_hexagon_V6_vrmpyub_rtt_acc_128B : +Hexagon_V65_v2048v2048v1024LLi_Intrinsic<"HEXAGON_V6_vrmpyub_rtt_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybub_rtt,VD_ftype_VIDI,2) +// tag : V6_vrmpybub_rtt +def int_hexagon_V6_vrmpybub_rtt : +Hexagon_V65_v1024v512LLi_Intrinsic<"HEXAGON_V6_vrmpybub_rtt">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybub_rtt_128B,VD_ftype_VIDI,2) +// tag : V6_vrmpybub_rtt_128B +def int_hexagon_V6_vrmpybub_rtt_128B : +Hexagon_V65_v2048v1024LLi_Intrinsic<"HEXAGON_V6_vrmpybub_rtt_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybub_rtt_acc,VD_ftype_VDVIDI,3) +// tag : V6_vrmpybub_rtt_acc +def int_hexagon_V6_vrmpybub_rtt_acc : +Hexagon_V65_v1024v1024v512LLi_Intrinsic<"HEXAGON_V6_vrmpybub_rtt_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybub_rtt_acc_128B,VD_ftype_VDVIDI,3) +// tag : V6_vrmpybub_rtt_acc_128B +def int_hexagon_V6_vrmpybub_rtt_acc_128B : +Hexagon_V65_v2048v2048v1024LLi_Intrinsic<"HEXAGON_V6_vrmpybub_rtt_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruwuhsat,VI_ftype_VIVISI,3) +// tag : V6_vasruwuhsat +def int_hexagon_V6_vasruwuhsat : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vasruwuhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruwuhsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasruwuhsat_128B +def int_hexagon_V6_vasruwuhsat_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasruwuhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruhubsat,VI_ftype_VIVISI,3) +// tag : V6_vasruhubsat +def int_hexagon_V6_vasruhubsat : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vasruhubsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruhubsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasruhubsat_128B +def int_hexagon_V6_vasruhubsat_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasruhubsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruhubrndsat,VI_ftype_VIVISI,3) +// tag : V6_vasruhubrndsat +def int_hexagon_V6_vasruhubrndsat : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vasruhubrndsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasruhubrndsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasruhubrndsat_128B +def int_hexagon_V6_vasruhubrndsat_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasruhubrndsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslh_acc,VI_ftype_VIVISI,3) +// tag : V6_vaslh_acc +def int_hexagon_V6_vaslh_acc : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vaslh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslh_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vaslh_acc_128B +def int_hexagon_V6_vaslh_acc_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vaslh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrh_acc,VI_ftype_VIVISI,3) +// tag : V6_vasrh_acc +def int_hexagon_V6_vasrh_acc : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrh_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrh_acc_128B +def int_hexagon_V6_vasrh_acc_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguw,VI_ftype_VIVI,2) +// tag : V6_vavguw +def int_hexagon_V6_vavguw : +Hexagon_V65_v512v512v512_Intrinsic<"HEXAGON_V6_vavguw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguw_128B,VI_ftype_VIVI,2) +// tag : V6_vavguw_128B +def int_hexagon_V6_vavguw_128B : +Hexagon_V65_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavguw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguwrnd,VI_ftype_VIVI,2) +// tag : V6_vavguwrnd +def int_hexagon_V6_vavguwrnd : +Hexagon_V65_v512v512v512_Intrinsic<"HEXAGON_V6_vavguwrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguwrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavguwrnd_128B +def int_hexagon_V6_vavguwrnd_128B : +Hexagon_V65_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavguwrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgb,VI_ftype_VIVI,2) +// tag : V6_vavgb +def int_hexagon_V6_vavgb : +Hexagon_V65_v512v512v512_Intrinsic<"HEXAGON_V6_vavgb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgb_128B,VI_ftype_VIVI,2) +// tag : V6_vavgb_128B +def int_hexagon_V6_vavgb_128B : +Hexagon_V65_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgbrnd,VI_ftype_VIVI,2) +// tag : V6_vavgbrnd +def int_hexagon_V6_vavgbrnd : +Hexagon_V65_v512v512v512_Intrinsic<"HEXAGON_V6_vavgbrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgbrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavgbrnd_128B +def int_hexagon_V6_vavgbrnd_128B : +Hexagon_V65_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgbrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgb,VI_ftype_VIVI,2) +// tag : V6_vnavgb +def int_hexagon_V6_vnavgb : +Hexagon_V65_v512v512v512_Intrinsic<"HEXAGON_V6_vnavgb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgb_128B,VI_ftype_VIVI,2) +// tag : V6_vnavgb_128B +def int_hexagon_V6_vnavgb_128B : +Hexagon_V65_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vnavgb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsb,VI_ftype_VI,1) +// tag : V6_vabsb +def int_hexagon_V6_vabsb : +Hexagon_V65_v512v512_Intrinsic<"HEXAGON_V6_vabsb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsb_128B,VI_ftype_VI,1) +// tag : V6_vabsb_128B +def int_hexagon_V6_vabsb_128B : +Hexagon_V65_v1024v1024_Intrinsic<"HEXAGON_V6_vabsb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsb_sat,VI_ftype_VI,1) +// tag : V6_vabsb_sat +def int_hexagon_V6_vabsb_sat : +Hexagon_V65_v512v512_Intrinsic<"HEXAGON_V6_vabsb_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsb_sat_128B,VI_ftype_VI,1) +// tag : V6_vabsb_sat_128B +def int_hexagon_V6_vabsb_sat_128B : +Hexagon_V65_v1024v1024_Intrinsic<"HEXAGON_V6_vabsb_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuu,VD_ftype_VDSI,2) +// tag : V6_vmpabuu +def int_hexagon_V6_vmpabuu : +Hexagon_V65_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpabuu">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuu_128B,VD_ftype_VDSI,2) +// tag : V6_vmpabuu_128B +def int_hexagon_V6_vmpabuu_128B : +Hexagon_V65_v2048v2048i_Intrinsic<"HEXAGON_V6_vmpabuu_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuu_acc,VD_ftype_VDVDSI,3) +// tag : V6_vmpabuu_acc +def int_hexagon_V6_vmpabuu_acc : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpabuu_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuu_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vmpabuu_acc_128B +def int_hexagon_V6_vmpabuu_acc_128B : +Hexagon_V65_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vmpabuu_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyh_acc,VD_ftype_VDVISI,3) +// tag : V6_vmpyh_acc +def int_hexagon_V6_vmpyh_acc : +Hexagon_V65_v1024v1024v512i_Intrinsic<"HEXAGON_V6_vmpyh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyh_acc_128B,VD_ftype_VDVISI,3) +// tag : V6_vmpyh_acc_128B +def int_hexagon_V6_vmpyh_acc_128B : +Hexagon_V65_v2048v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahhsat,VI_ftype_VIVIDI,3) +// tag : V6_vmpahhsat +def int_hexagon_V6_vmpahhsat : +Hexagon_V65_v512v512v512LLi_Intrinsic<"HEXAGON_V6_vmpahhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahhsat_128B,VI_ftype_VIVIDI,3) +// tag : V6_vmpahhsat_128B +def int_hexagon_V6_vmpahhsat_128B : +Hexagon_V65_v1024v1024v1024LLi_Intrinsic<"HEXAGON_V6_vmpahhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpauhuhsat,VI_ftype_VIVIDI,3) +// tag : V6_vmpauhuhsat +def int_hexagon_V6_vmpauhuhsat : +Hexagon_V65_v512v512v512LLi_Intrinsic<"HEXAGON_V6_vmpauhuhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpauhuhsat_128B,VI_ftype_VIVIDI,3) +// tag : V6_vmpauhuhsat_128B +def int_hexagon_V6_vmpauhuhsat_128B : +Hexagon_V65_v1024v1024v1024LLi_Intrinsic<"HEXAGON_V6_vmpauhuhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpsuhuhsat,VI_ftype_VIVIDI,3) +// tag : V6_vmpsuhuhsat +def int_hexagon_V6_vmpsuhuhsat : +Hexagon_V65_v512v512v512LLi_Intrinsic<"HEXAGON_V6_vmpsuhuhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpsuhuhsat_128B,VI_ftype_VIVIDI,3) +// tag : V6_vmpsuhuhsat_128B +def int_hexagon_V6_vmpsuhuhsat_128B : +Hexagon_V65_v1024v1024v1024LLi_Intrinsic<"HEXAGON_V6_vmpsuhuhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlut4,VI_ftype_VIDI,2) +// tag : V6_vlut4 +def int_hexagon_V6_vlut4 : +Hexagon_V65_v512v512LLi_Intrinsic<"HEXAGON_V6_vlut4">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlut4_128B,VI_ftype_VIDI,2) +// tag : V6_vlut4_128B +def int_hexagon_V6_vlut4_128B : +Hexagon_V65_v1024v1024LLi_Intrinsic<"HEXAGON_V6_vlut4_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhe,VI_ftype_VISI,2) +// tag : V6_vmpyuhe +def int_hexagon_V6_vmpyuhe : +Hexagon_V65_v512v512i_Intrinsic<"HEXAGON_V6_vmpyuhe">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhe_128B,VI_ftype_VISI,2) +// tag : V6_vmpyuhe_128B +def int_hexagon_V6_vmpyuhe_128B : +Hexagon_V65_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyuhe_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhe_acc,VI_ftype_VIVISI,3) +// tag : V6_vmpyuhe_acc +def int_hexagon_V6_vmpyuhe_acc : +Hexagon_V65_v512v512v512i_Intrinsic<"HEXAGON_V6_vmpyuhe_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhe_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vmpyuhe_acc_128B +def int_hexagon_V6_vmpyuhe_acc_128B : +Hexagon_V65_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyuhe_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqb,VI_ftype_QV,1) +// tag : V6_vprefixqb +def int_hexagon_V6_vprefixqb : +Hexagon_V65_v512v64i_Intrinsic<"HEXAGON_V6_vprefixqb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqb_128B,VI_ftype_QV,1) +// tag : V6_vprefixqb_128B +def int_hexagon_V6_vprefixqb_128B : +Hexagon_V65_v1024v128i_Intrinsic<"HEXAGON_V6_vprefixqb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqh,VI_ftype_QV,1) +// tag : V6_vprefixqh +def int_hexagon_V6_vprefixqh : +Hexagon_V65_v512v64i_Intrinsic<"HEXAGON_V6_vprefixqh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqh_128B,VI_ftype_QV,1) +// tag : V6_vprefixqh_128B +def int_hexagon_V6_vprefixqh_128B : +Hexagon_V65_v1024v128i_Intrinsic<"HEXAGON_V6_vprefixqh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqw,VI_ftype_QV,1) +// tag : V6_vprefixqw +def int_hexagon_V6_vprefixqw : +Hexagon_V65_v512v64i_Intrinsic<"HEXAGON_V6_vprefixqw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vprefixqw_128B,VI_ftype_QV,1) +// tag : V6_vprefixqw_128B +def int_hexagon_V6_vprefixqw_128B : +Hexagon_V65_v1024v128i_Intrinsic<"HEXAGON_V6_vprefixqw_128B">; + + +// The scatter/gather ones below will not be generated from iset.py. Make sure +// you don't overwrite these. +class Hexagon_V65_vvmemiiv512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty, + llvm_v16i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemiiv1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty, + llvm_v32i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemiiv2048_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty, + llvm_v64i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemv64iiiv512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_v512i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v16i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemv128iiiv1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_v1024i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v32i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemv64iiiv1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_v512i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v32i32_ty], + [IntrArgMemOnly]>; + +class Hexagon_V65_vvmemv128iiiv2048_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_ptr_ty,llvm_v1024i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v64i32_ty], + [IntrArgMemOnly]>; + +def int_hexagon_V6_vgathermw : +Hexagon_V65_vvmemiiv512_Intrinsic<"HEXAGON_V6_vgathermw">; + +def int_hexagon_V6_vgathermw_128B : +Hexagon_V65_vvmemiiv1024_Intrinsic<"HEXAGON_V6_vgathermw_128B">; + +def int_hexagon_V6_vgathermh : +Hexagon_V65_vvmemiiv512_Intrinsic<"HEXAGON_V6_vgathermh">; + +def int_hexagon_V6_vgathermh_128B : +Hexagon_V65_vvmemiiv1024_Intrinsic<"HEXAGON_V6_vgathermh_128B">; + +def int_hexagon_V6_vgathermhw : +Hexagon_V65_vvmemiiv1024_Intrinsic<"HEXAGON_V6_vgathermhw">; + +def int_hexagon_V6_vgathermhw_128B : +Hexagon_V65_vvmemiiv2048_Intrinsic<"HEXAGON_V6_vgathermhw_128B">; + +def int_hexagon_V6_vgathermwq : +Hexagon_V65_vvmemv64iiiv512_Intrinsic<"HEXAGON_V6_vgathermwq">; + +def int_hexagon_V6_vgathermwq_128B : +Hexagon_V65_vvmemv128iiiv1024_Intrinsic<"HEXAGON_V6_vgathermwq_128B">; + +def int_hexagon_V6_vgathermhq : +Hexagon_V65_vvmemv64iiiv512_Intrinsic<"HEXAGON_V6_vgathermhq">; + +def int_hexagon_V6_vgathermhq_128B : +Hexagon_V65_vvmemv128iiiv1024_Intrinsic<"HEXAGON_V6_vgathermhq_128B">; + +def int_hexagon_V6_vgathermhwq : +Hexagon_V65_vvmemv64iiiv1024_Intrinsic<"HEXAGON_V6_vgathermhwq">; + +def int_hexagon_V6_vgathermhwq_128B : +Hexagon_V65_vvmemv128iiiv2048_Intrinsic<"HEXAGON_V6_vgathermhwq_128B">; + +class Hexagon_V65_viiv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_i32_ty,llvm_i32_ty, + llvm_v16i32_ty,llvm_v16i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_viiv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_i32_ty,llvm_i32_ty, + llvm_v32i32_ty,llvm_v32i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_vv64iiiv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_v512i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v16i32_ty, + llvm_v16i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_vv128iiiv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_v1024i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v32i32_ty, + llvm_v32i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_viiv1024v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_i32_ty,llvm_i32_ty, + llvm_v32i32_ty,llvm_v16i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_viiv2048v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_i32_ty,llvm_i32_ty, + llvm_v64i32_ty,llvm_v32i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_vv64iiiv1024v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_v512i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v32i32_ty, + llvm_v16i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_vv128iiiv2048v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [], [llvm_v1024i1_ty,llvm_i32_ty, + llvm_i32_ty,llvm_v64i32_ty, + llvm_v32i32_ty], + [IntrWriteMem]>; + +class Hexagon_V65_v2048_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [], + [IntrNoMem]>; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermw,v_ftype_SISIVIVI,4) +// tag : V6_vscattermw +def int_hexagon_V6_vscattermw : +Hexagon_V65_viiv512v512_Intrinsic<"HEXAGON_V6_vscattermw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermw_128B,v_ftype_SISIVIVI,4) +// tag : V6_vscattermw_128B +def int_hexagon_V6_vscattermw_128B : +Hexagon_V65_viiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermh,v_ftype_SISIVIVI,4) +// tag : V6_vscattermh +def int_hexagon_V6_vscattermh : +Hexagon_V65_viiv512v512_Intrinsic<"HEXAGON_V6_vscattermh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermh_128B,v_ftype_SISIVIVI,4) +// tag : V6_vscattermh_128B +def int_hexagon_V6_vscattermh_128B : +Hexagon_V65_viiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermw_add,v_ftype_SISIVIVI,4) +// tag : V6_vscattermw_add +def int_hexagon_V6_vscattermw_add : +Hexagon_V65_viiv512v512_Intrinsic<"HEXAGON_V6_vscattermw_add">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermw_add_128B,v_ftype_SISIVIVI,4) +// tag : V6_vscattermw_add_128B +def int_hexagon_V6_vscattermw_add_128B : +Hexagon_V65_viiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermw_add_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermh_add,v_ftype_SISIVIVI,4) +// tag : V6_vscattermh_add +def int_hexagon_V6_vscattermh_add : +Hexagon_V65_viiv512v512_Intrinsic<"HEXAGON_V6_vscattermh_add">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermh_add_128B,v_ftype_SISIVIVI,4) +// tag : V6_vscattermh_add_128B +def int_hexagon_V6_vscattermh_add_128B : +Hexagon_V65_viiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermh_add_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermwq,v_ftype_QVSISIVIVI,5) +// tag : V6_vscattermwq +def int_hexagon_V6_vscattermwq : +Hexagon_V65_vv64iiiv512v512_Intrinsic<"HEXAGON_V6_vscattermwq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermwq_128B,v_ftype_QVSISIVIVI,5) +// tag : V6_vscattermwq_128B +def int_hexagon_V6_vscattermwq_128B : +Hexagon_V65_vv128iiiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermwq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhq,v_ftype_QVSISIVIVI,5) +// tag : V6_vscattermhq +def int_hexagon_V6_vscattermhq : +Hexagon_V65_vv64iiiv512v512_Intrinsic<"HEXAGON_V6_vscattermhq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhq_128B,v_ftype_QVSISIVIVI,5) +// tag : V6_vscattermhq_128B +def int_hexagon_V6_vscattermhq_128B : +Hexagon_V65_vv128iiiv1024v1024_Intrinsic<"HEXAGON_V6_vscattermhq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhw,v_ftype_SISIVDVI,4) +// tag : V6_vscattermhw +def int_hexagon_V6_vscattermhw : +Hexagon_V65_viiv1024v512_Intrinsic<"HEXAGON_V6_vscattermhw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhw_128B,v_ftype_SISIVDVI,4) +// tag : V6_vscattermhw_128B +def int_hexagon_V6_vscattermhw_128B : +Hexagon_V65_viiv2048v1024_Intrinsic<"HEXAGON_V6_vscattermhw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhwq,v_ftype_QVSISIVDVI,5) +// tag : V6_vscattermhwq +def int_hexagon_V6_vscattermhwq : +Hexagon_V65_vv64iiiv1024v512_Intrinsic<"HEXAGON_V6_vscattermhwq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhwq_128B,v_ftype_QVSISIVDVI,5) +// tag : V6_vscattermhwq_128B +def int_hexagon_V6_vscattermhwq_128B : +Hexagon_V65_vv128iiiv2048v1024_Intrinsic<"HEXAGON_V6_vscattermhwq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhw_add,v_ftype_SISIVDVI,4) +// tag : V6_vscattermhw_add +def int_hexagon_V6_vscattermhw_add : +Hexagon_V65_viiv1024v512_Intrinsic<"HEXAGON_V6_vscattermhw_add">; + +// +// BUILTIN_INFO(HEXAGON.V6_vscattermhw_add_128B,v_ftype_SISIVDVI,4) +// tag : V6_vscattermhw_add_128B +def int_hexagon_V6_vscattermhw_add_128B : +Hexagon_V65_viiv2048v1024_Intrinsic<"HEXAGON_V6_vscattermhw_add_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdd0,VD_ftype_,0) +// tag : V6_vdd0 +def int_hexagon_V6_vdd0 : +Hexagon_v1024_Intrinsic<"HEXAGON_V6_vdd0">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdd0_128B,VD_ftype_,0) +// tag : V6_vdd0_128B +def int_hexagon_V6_vdd0_128B : +Hexagon_V65_v2048_Intrinsic<"HEXAGON_V6_vdd0_128B">; diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td b/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td index 68f123df0430..73622ce9303f 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -682,11 +682,21 @@ let TargetPrefix = "nvvm" in { def int_nvvm_bitcast_d2ll : GCCBuiltin<"__nvvm_bitcast_d2ll">, Intrinsic<[llvm_i64_ty], [llvm_double_ty], [IntrNoMem]>; +// FNS -// Atomic not available as an llvm intrinsic. + def int_nvvm_fns : GCCBuiltin<"__nvvm_fns">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + +// Atomics not available as llvm intrinsics. def int_nvvm_atomic_load_add_f32 : Intrinsic<[llvm_float_ty], [LLVMAnyPointerType<llvm_float_ty>, llvm_float_ty], [IntrArgMemOnly, NoCapture<0>]>; + // Atomic add of f64 requires sm_60. + def int_nvvm_atomic_load_add_f64 : Intrinsic<[llvm_double_ty], + [LLVMAnyPointerType<llvm_double_ty>, llvm_double_ty], + [IntrArgMemOnly, NoCapture<0>]>; + def int_nvvm_atomic_load_inc_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType<llvm_i32_ty>, llvm_i32_ty], [IntrArgMemOnly, NoCapture<0>]>; @@ -750,6 +760,17 @@ let TargetPrefix = "nvvm" in { def int_nvvm_bar_sync : Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, GCCBuiltin<"__nvvm_bar_sync">; + def int_nvvm_bar_warp_sync : + Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_bar_warp_sync">; + + // barrier.sync id[, cnt] + def int_nvvm_barrier_sync : + Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_barrier_sync">; + def int_nvvm_barrier_sync_cnt : + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_barrier_sync_cnt">; // Membar def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">, @@ -3700,40 +3721,308 @@ def int_nvvm_read_ptx_sreg_warpsize : PTXReadSRegIntrinsic_r32<"warpsize">; // shfl.down.b32 dest, val, offset, mask_and_clamp def int_nvvm_shfl_down_i32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.down.i32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.down.i32">, GCCBuiltin<"__nvvm_shfl_down_i32">; def int_nvvm_shfl_down_f32 : Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.down.f32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.down.f32">, GCCBuiltin<"__nvvm_shfl_down_f32">; // shfl.up.b32 dest, val, offset, mask_and_clamp def int_nvvm_shfl_up_i32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.up.i32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.up.i32">, GCCBuiltin<"__nvvm_shfl_up_i32">; def int_nvvm_shfl_up_f32 : Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.up.f32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.up.f32">, GCCBuiltin<"__nvvm_shfl_up_f32">; // shfl.bfly.b32 dest, val, offset, mask_and_clamp def int_nvvm_shfl_bfly_i32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.bfly.i32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.bfly.i32">, GCCBuiltin<"__nvvm_shfl_bfly_i32">; def int_nvvm_shfl_bfly_f32 : Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.bfly.f32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.bfly.f32">, GCCBuiltin<"__nvvm_shfl_bfly_f32">; // shfl.idx.b32 dest, val, lane, mask_and_clamp def int_nvvm_shfl_idx_i32 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.idx.i32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.idx.i32">, GCCBuiltin<"__nvvm_shfl_idx_i32">; def int_nvvm_shfl_idx_f32 : Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.idx.f32">, + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.idx.f32">, GCCBuiltin<"__nvvm_shfl_idx_f32">; + +// Synchronizing shfl variants available in CUDA-9. +// On sm_70 these don't have to be convergent, so we may eventually want to +// implement non-convergent variant of this intrinsic. + +// shfl.sync.down.b32 dest, threadmask, val, offset , mask_and_clamp +def int_nvvm_shfl_sync_down_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.down.i32">, + GCCBuiltin<"__nvvm_shfl_sync_down_i32">; +def int_nvvm_shfl_sync_down_f32 : + Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.down.f32">, + GCCBuiltin<"__nvvm_shfl_sync_down_f32">; + +// shfl.sync.up.b32 dest, threadmask, val, offset, mask_and_clamp +def int_nvvm_shfl_sync_up_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.up.i32">, + GCCBuiltin<"__nvvm_shfl_sync_up_i32">; +def int_nvvm_shfl_sync_up_f32 : + Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.up.f32">, + GCCBuiltin<"__nvvm_shfl_sync_up_f32">; + +// shfl.sync.bfly.b32 dest, threadmask, val, offset, mask_and_clamp +def int_nvvm_shfl_sync_bfly_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.bfly.i32">, + GCCBuiltin<"__nvvm_shfl_sync_bfly_i32">; +def int_nvvm_shfl_sync_bfly_f32 : + Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.bfly.f32">, + GCCBuiltin<"__nvvm_shfl_sync_bfly_f32">; + +// shfl.sync.idx.b32 dest, threadmask, val, lane, mask_and_clamp +def int_nvvm_shfl_sync_idx_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.idx.i32">, + GCCBuiltin<"__nvvm_shfl_sync_idx_i32">; +def int_nvvm_shfl_sync_idx_f32 : + Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.shfl.sync.idx.f32">, + GCCBuiltin<"__nvvm_shfl_sync_idx_f32">; + +// +// VOTE +// + +// vote.all pred +def int_nvvm_vote_all : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.all">, + GCCBuiltin<"__nvvm_vote_all">; +// vote.any pred +def int_nvvm_vote_any : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.any">, + GCCBuiltin<"__nvvm_vote_any">; +// vote.uni pred +def int_nvvm_vote_uni : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.uni">, + GCCBuiltin<"__nvvm_vote_uni">; +// vote.ballot pred +def int_nvvm_vote_ballot : + Intrinsic<[llvm_i32_ty], [llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.ballot">, + GCCBuiltin<"__nvvm_vote_ballot">; + +// +// VOTE.SYNC +// + +// vote.sync.all mask, pred +def int_nvvm_vote_all_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.all.sync">, + GCCBuiltin<"__nvvm_vote_all_sync">; +// vote.sync.any mask, pred +def int_nvvm_vote_any_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.any.sync">, + GCCBuiltin<"__nvvm_vote_any_sync">; +// vote.sync.uni mask, pred +def int_nvvm_vote_uni_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.uni.sync">, + GCCBuiltin<"__nvvm_vote_uni_sync">; +// vote.sync.ballot mask, pred +def int_nvvm_vote_ballot_sync : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.vote.ballot.sync">, + GCCBuiltin<"__nvvm_vote_ballot_sync">; + +// +// MATCH.SYNC +// +// match.any.sync.b32 mask, value +def int_nvvm_match_any_sync_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.any.sync.i32">, + GCCBuiltin<"__nvvm_match_any_sync_i32">; +// match.any.sync.b64 mask, value +def int_nvvm_match_any_sync_i64 : + Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.any.sync.i64">, + GCCBuiltin<"__nvvm_match_any_sync_i64">; + +// match.all instruction have two variants -- one returns a single value, another +// returns a pair {value, predicate}. We currently only implement the latter as +// that's the variant exposed by CUDA API. + +// match.all.sync.b32p mask, value +def int_nvvm_match_all_sync_i32p : + Intrinsic<[llvm_i32_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.all.sync.i32p">; +// match.all.sync.b64p mask, value +def int_nvvm_match_all_sync_i64p : + Intrinsic<[llvm_i64_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.all.sync.i64p">; + +// +// WMMA instructions +// + +// WMMA.LOAD +class NVVM_WMMA_LD_ALSTS<string Abc, string Layout, string Space, + string Type, LLVMType regty, int WithStride> + : Intrinsic<!if(!eq(Abc#Type,"cf16"), + [regty, regty, regty, regty], + [regty, regty, regty, regty, + regty, regty, regty, regty]), + !if(WithStride, [llvm_ptr_ty, llvm_i32_ty], [llvm_ptr_ty]), + [], // Properties must be set during instantiation. + "llvm.nvvm.wmma.load."#Abc#".sync."#Layout#".m16n16k16" + #Space + #!if(WithStride,".stride","") + #"."#Type>; + +multiclass NVVM_WMMA_LD_ALST<string Abc, string Layout, string Space, + string Type, LLVMType regty> { + def _stride: NVVM_WMMA_LD_ALSTS<Abc, Layout, Space, Type, regty, 1>; + def NAME : NVVM_WMMA_LD_ALSTS<Abc, Layout, Space, Type, regty, 0>; } + +multiclass NVVM_WMMA_LD_ALT<string Abc, string Layout, + string Type, LLVMType regty> { + defm _global: NVVM_WMMA_LD_ALST<Abc, Layout, ".global", Type, regty>; + defm _shared: NVVM_WMMA_LD_ALST<Abc, Layout, ".shared", Type, regty>; + defm NAME: NVVM_WMMA_LD_ALST<Abc, Layout, "", Type, regty>; +} + +multiclass NVVM_WMMA_LD_AT<string Abc, string Type, LLVMType regty> { + defm _row: NVVM_WMMA_LD_ALT<Abc, "row", Type, regty>; + defm _col: NVVM_WMMA_LD_ALT<Abc, "col", Type, regty>; +} + +// For some reason ReadOnly<N> and NoCapture<N> confuses tblgen if they are +// passed to Intrinsic<> form inside of a multiclass. Setting them globally +// outside of the multiclass works. +let IntrProperties = [IntrReadMem, IntrArgMemOnly, + ReadOnly<0>, NoCapture<0>] in { + defm int_nvvm_wmma_load_a_f16: NVVM_WMMA_LD_AT<"a", "f16", llvm_v2f16_ty>; + defm int_nvvm_wmma_load_b_f16: NVVM_WMMA_LD_AT<"b", "f16", llvm_v2f16_ty>; + defm int_nvvm_wmma_load_c_f16: NVVM_WMMA_LD_AT<"c", "f16", llvm_v2f16_ty>; + defm int_nvvm_wmma_load_c_f32: NVVM_WMMA_LD_AT<"c", "f32", llvm_float_ty>; +} + +// WMMA.STORE.D +class NVVM_WMMA_STD_LSTS<string Layout, string Space, + string Type, LLVMType regty, int WithStride, + // This is only used to create a typed empty array we + // need to pass to !if below. + list<LLVMType>Empty=[]> + : Intrinsic<[], + !listconcat( + [llvm_ptr_ty], + !if(!eq(Type,"f16"), + [regty, regty, regty, regty], + [regty, regty, regty, regty, + regty, regty, regty, regty]), + !if(WithStride, [llvm_i32_ty], Empty)), + [], // Properties must be set during instantiation. + "llvm.nvvm.wmma.store.d.sync."#Layout + #".m16n16k16"#Space + #!if(WithStride,".stride","") + #"."#Type>; + +multiclass NVVM_WMMA_STD_LST<string Layout, string Space, + string Type, LLVMType regty> { + def _stride: NVVM_WMMA_STD_LSTS<Layout, Space, Type, regty, 1>; + def NAME: NVVM_WMMA_STD_LSTS<Layout, Space, Type, regty, 0>; +} + +multiclass NVVM_WMMA_STD_LT<string Layout, string Type, LLVMType regty> { + defm _global: NVVM_WMMA_STD_LST<Layout, ".global", Type, regty>; + defm _shared: NVVM_WMMA_STD_LST<Layout, ".shared", Type, regty>; + defm NAME: NVVM_WMMA_STD_LST<Layout, "", Type, regty>; +} + +multiclass NVVM_WMMA_STD_T<string Type, LLVMType regty> { + defm _row: NVVM_WMMA_STD_LT<"row", Type, regty>; + defm _col: NVVM_WMMA_STD_LT<"col", Type, regty>; +} + +let IntrProperties = [IntrWriteMem, IntrArgMemOnly, + WriteOnly<0>, NoCapture<0>] in { + defm int_nvvm_wmma_store_d_f16: NVVM_WMMA_STD_T<"f16", llvm_v2f16_ty>; + defm int_nvvm_wmma_store_d_f32: NVVM_WMMA_STD_T<"f32", llvm_float_ty>; +} + +// WMMA.MMA +class NVVM_WMMA_MMA_ABDCS<string ALayout, string BLayout, + string DType, LLVMType d_regty, + string CType, LLVMType c_regty, + string Satfinite = ""> + : Intrinsic<!if(!eq(DType,"f16"), + [d_regty, d_regty, d_regty, d_regty], + [d_regty, d_regty, d_regty, d_regty, + d_regty, d_regty, d_regty, d_regty]), + !listconcat( + [// A + llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, + llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, + // B + llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, + llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty], + !if(!eq(CType,"f16"), + [c_regty, c_regty, c_regty, c_regty], + [c_regty, c_regty, c_regty, c_regty, + c_regty, c_regty, c_regty, c_regty])), + [IntrNoMem], + "llvm.nvvm.wmma.mma.sync."#ALayout#"."#BLayout + #".m16n16k16."#DType#"."#CType#Satfinite>; + +multiclass NVVM_WMMA_MMA_ABDC<string ALayout, string BLayout, + string DType, LLVMType d_regty, + string CType, LLVMType c_regty> { + def NAME : NVVM_WMMA_MMA_ABDCS<ALayout, BLayout, + DType, d_regty, + CType, c_regty>; + def _satfinite: NVVM_WMMA_MMA_ABDCS<ALayout, BLayout, + DType, d_regty, + CType, c_regty,".satfinite">; +} + +multiclass NVVM_WMMA_MMA_ABD<string ALayout, string BLayout, + string DType, LLVMType d_regty> { + defm _f16: NVVM_WMMA_MMA_ABDC<ALayout, BLayout, DType, d_regty, + "f16", llvm_v2f16_ty>; + defm _f32: NVVM_WMMA_MMA_ABDC<ALayout, BLayout, DType, d_regty, + "f32", llvm_float_ty>; +} + +multiclass NVVM_WMMA_MMA_AB<string ALayout, string BLayout> { + defm _f16: NVVM_WMMA_MMA_ABD<ALayout, BLayout, "f16", llvm_v2f16_ty>; + defm _f32: NVVM_WMMA_MMA_ABD<ALayout, BLayout, "f32", llvm_float_ty>; +} + +multiclass NVVM_WMMA_MMA_A<string ALayout> { + defm _col: NVVM_WMMA_MMA_AB<ALayout, "col">; + defm _row: NVVM_WMMA_MMA_AB<ALayout, "row">; +} + +defm int_nvvm_wmma_mma_sync_col: NVVM_WMMA_MMA_A<"col">; +defm int_nvvm_wmma_mma_sync_row: NVVM_WMMA_MMA_A<"row">; + +} // let TargetPrefix = "nvvm" diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsSystemZ.td b/contrib/llvm/include/llvm/IR/IntrinsicsSystemZ.td index 98065bc51d99..caa2ec209a31 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsSystemZ.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsSystemZ.td @@ -198,17 +198,17 @@ multiclass SystemZQuaternaryIntCCBHF { let TargetPrefix = "s390" in { def int_s390_tbegin : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrNoDuplicate]>; + [IntrNoDuplicate, IntrWriteMem]>; def int_s390_tbegin_nofloat : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrNoDuplicate]>; + [IntrNoDuplicate, IntrWriteMem]>; def int_s390_tbeginc : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], - [IntrNoDuplicate]>; + [IntrNoDuplicate, IntrWriteMem]>; def int_s390_tabort : Intrinsic<[], [llvm_i64_ty], - [IntrNoReturn, Throws]>; + [IntrNoReturn, Throws, IntrWriteMem]>; def int_s390_tend : GCCBuiltin<"__builtin_tend">, Intrinsic<[llvm_i32_ty], []>; @@ -217,7 +217,7 @@ let TargetPrefix = "s390" in { Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; def int_s390_ntstg : Intrinsic<[], [llvm_i64_ty, llvm_ptr64_ty], - [IntrArgMemOnly]>; + [IntrArgMemOnly, IntrWriteMem]>; def int_s390_ppa_txassist : GCCBuiltin<"__builtin_tx_assist">, Intrinsic<[], [llvm_i32_ty]>; @@ -260,9 +260,7 @@ let TargetPrefix = "s390" in { def int_s390_vstl : GCCBuiltin<"__builtin_s390_vstl">, Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], - // In fact write-only but there's no property - // for that. - [IntrArgMemOnly]>; + [IntrArgMemOnly, IntrWriteMem]>; defm int_s390_vupl : SystemZUnaryExtBHWF<"vupl">; defm int_s390_vupll : SystemZUnaryExtBHF<"vupll">; @@ -413,9 +411,7 @@ let TargetPrefix = "s390" in { def int_s390_vstrl : GCCBuiltin<"__builtin_s390_vstrl">, Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], - // In fact write-only but there's no property - // for that. - [IntrArgMemOnly]>; + [IntrArgMemOnly, IntrWriteMem]>; } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/include/llvm/IR/IntrinsicsX86.td b/contrib/llvm/include/llvm/IR/IntrinsicsX86.td index 80c528768dc7..bd6177c5b3d9 100644 --- a/contrib/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/contrib/llvm/include/llvm/IR/IntrinsicsX86.td @@ -64,6 +64,35 @@ let TargetPrefix = "x86" in { } //===----------------------------------------------------------------------===// +// CET SS +let TargetPrefix = "x86" in { + def int_x86_incsspd : GCCBuiltin<"__builtin_ia32_incsspd">, + Intrinsic<[], [llvm_i32_ty], []>; + def int_x86_incsspq : GCCBuiltin<"__builtin_ia32_incsspq">, + Intrinsic<[], [llvm_i64_ty], []>; + def int_x86_rdsspd : GCCBuiltin<"__builtin_ia32_rdsspd">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; + def int_x86_rdsspq : GCCBuiltin<"__builtin_ia32_rdsspq">, + Intrinsic<[llvm_i64_ty], [llvm_i64_ty], []>; + def int_x86_saveprevssp : GCCBuiltin<"__builtin_ia32_saveprevssp">, + Intrinsic<[], [], []>; + def int_x86_rstorssp : GCCBuiltin<"__builtin_ia32_rstorssp">, + Intrinsic<[], [llvm_ptr_ty], []>; + def int_x86_wrssd : GCCBuiltin<"__builtin_ia32_wrssd">, + Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; + def int_x86_wrssq : GCCBuiltin<"__builtin_ia32_wrssq">, + Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>; + def int_x86_wrussd : GCCBuiltin<"__builtin_ia32_wrussd">, + Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; + def int_x86_wrussq : GCCBuiltin<"__builtin_ia32_wrussq">, + Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], []>; + def int_x86_setssbsy : GCCBuiltin<"__builtin_ia32_setssbsy">, + Intrinsic<[], [], []>; + def int_x86_clrssbsy : GCCBuiltin<"__builtin_ia32_clrssbsy">, + Intrinsic<[], [llvm_ptr_ty], []>; +} + +//===----------------------------------------------------------------------===// // 3DNow! let TargetPrefix = "x86" in { @@ -379,12 +408,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, - llvm_v16i8_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_v8i16_ty], [IntrNoMem, Commutative]>; def int_x86_sse2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw128">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, Commutative]>; @@ -664,18 +687,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_ssse3_pabs_b : GCCBuiltin<"__builtin_ia32_pabsb">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pabs_b_128 : GCCBuiltin<"__builtin_ia32_pabsb128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; def int_x86_ssse3_pabs_w : GCCBuiltin<"__builtin_ia32_pabsw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pabs_w_128 : GCCBuiltin<"__builtin_ia32_pabsw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_ssse3_pabs_d : GCCBuiltin<"__builtin_ia32_pabsd">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty], [IntrNoMem]>; - def int_x86_ssse3_pabs_d_128 : GCCBuiltin<"__builtin_ia32_pabsd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; } //===----------------------------------------------------------------------===// @@ -709,29 +726,68 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_aesni_aesimc : GCCBuiltin<"__builtin_ia32_aesimc128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_aesni_aesenc : GCCBuiltin<"__builtin_ia32_aesenc128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_aesni_aesenc_256 : GCCBuiltin<"__builtin_ia32_aesenc256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesenc_512 : GCCBuiltin<"__builtin_ia32_aesenc512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesenclast : GCCBuiltin<"__builtin_ia32_aesenclast128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_aesni_aesenclast_256 : + GCCBuiltin<"__builtin_ia32_aesenclast256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesenclast_512 : + GCCBuiltin<"__builtin_ia32_aesenclast512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesdec : GCCBuiltin<"__builtin_ia32_aesdec128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_aesni_aesdec_256 : GCCBuiltin<"__builtin_ia32_aesdec256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesdec_512 : GCCBuiltin<"__builtin_ia32_aesdec512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesdeclast : GCCBuiltin<"__builtin_ia32_aesdeclast128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_aesni_aesdeclast_256 : + GCCBuiltin<"__builtin_ia32_aesdeclast256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aesdeclast_512 : + GCCBuiltin<"__builtin_ia32_aesdeclast512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty], + [IntrNoMem]>; + def int_x86_aesni_aeskeygenassist : GCCBuiltin<"__builtin_ia32_aeskeygenassist128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; } -// PCLMUL instruction +// PCLMUL instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_pclmulqdq : GCCBuiltin<"__builtin_ia32_pclmulqdq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_pclmulqdq_256 : GCCBuiltin<"__builtin_ia32_pclmulqdq256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_pclmulqdq_512 : GCCBuiltin<"__builtin_ia32_pclmulqdq512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; } // Vector pack @@ -977,19 +1033,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_vpermilvarps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx_vperm2f128_pd_256 : - GCCBuiltin<"__builtin_ia32_vperm2f128_pd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx_vperm2f128_ps_256 : - GCCBuiltin<"__builtin_ia32_vperm2f128_ps256">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx_vperm2f128_si_256 : - GCCBuiltin<"__builtin_ia32_vperm2f128_si256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_d_128 : GCCBuiltin<"__builtin_ia32_vpermi2vard128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -1325,52 +1368,56 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_shuf_f32x4_256 : - GCCBuiltin<"__builtin_ia32_shuf_f32x4_256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; +} - def int_x86_avx512_mask_shuf_f32x4 : - GCCBuiltin<"__builtin_ia32_shuf_f32x4_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], +// GFNI Instructions +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_vgf2p8affineinvqb_128 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v16qi">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_shuf_f64x2_256 : - GCCBuiltin<"__builtin_ia32_shuf_f64x2_256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + def int_x86_vgf2p8affineinvqb_256 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v32qi">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_shuf_f64x2 : - GCCBuiltin<"__builtin_ia32_shuf_f64x2_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + def int_x86_vgf2p8affineinvqb_512 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v64qi">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_shuf_i32x4_256 : - GCCBuiltin<"__builtin_ia32_shuf_i32x4_256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], + def int_x86_vgf2p8affineqb_128 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v16qi">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_shuf_i32x4 : - GCCBuiltin<"__builtin_ia32_shuf_i32x4_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], + def int_x86_vgf2p8affineqb_256 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v32qi">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_shuf_i64x2_256 : - GCCBuiltin<"__builtin_ia32_shuf_i64x2_256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], + def int_x86_vgf2p8affineqb_512 : + GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v64qi">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_shuf_i64x2 : - GCCBuiltin<"__builtin_ia32_shuf_i64x2_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], + def int_x86_vgf2p8mulb_128 : + GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v16qi">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty], + [IntrNoMem]>; + def int_x86_vgf2p8mulb_256 : + GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v32qi">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty], + [IntrNoMem]>; + def int_x86_vgf2p8mulb_512 : + GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v64qi">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>; } @@ -1464,80 +1511,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_ptestnzc_256 : GCCBuiltin<"__builtin_ia32_ptestnzc256">, Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_d_512 : GCCBuiltin<"__builtin_ia32_ptestmd512">, - Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_q_512 : GCCBuiltin<"__builtin_ia32_ptestmq512">, - Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_ptestm_b_128 : GCCBuiltin<"__builtin_ia32_ptestmb128">, - Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, - llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_b_256 : GCCBuiltin<"__builtin_ia32_ptestmb256">, - Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, - llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_b_512 : GCCBuiltin<"__builtin_ia32_ptestmb512">, - Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, - llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_d_128 : GCCBuiltin<"__builtin_ia32_ptestmd128">, - Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_d_256 : GCCBuiltin<"__builtin_ia32_ptestmd256">, - Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_q_128 : GCCBuiltin<"__builtin_ia32_ptestmq128">, - Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_q_256 : GCCBuiltin<"__builtin_ia32_ptestmq256">, - Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_w_128 : GCCBuiltin<"__builtin_ia32_ptestmw128">, - Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, - llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_w_256 : GCCBuiltin<"__builtin_ia32_ptestmw256">, - Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestm_w_512 : GCCBuiltin<"__builtin_ia32_ptestmw512">, - Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, - llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_ptestnm_b_128 : GCCBuiltin<"__builtin_ia32_ptestnmb128">, - Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, - llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_b_256 : GCCBuiltin<"__builtin_ia32_ptestnmb256">, - Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, - llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_b_512 : GCCBuiltin<"__builtin_ia32_ptestnmb512">, - Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, - llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_d_128 : GCCBuiltin<"__builtin_ia32_ptestnmd128">, - Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_d_256 : GCCBuiltin<"__builtin_ia32_ptestnmd256">, - Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_d_512 : GCCBuiltin<"__builtin_ia32_ptestnmd512">, - Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_q_128 : GCCBuiltin<"__builtin_ia32_ptestnmq128">, - Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_q_256 : GCCBuiltin<"__builtin_ia32_ptestnmq256">, - Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_q_512 : GCCBuiltin<"__builtin_ia32_ptestnmq512">, - Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_w_128 : GCCBuiltin<"__builtin_ia32_ptestnmw128">, - Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, - llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_w_256 : GCCBuiltin<"__builtin_ia32_ptestnmw256">, - Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_ptestnm_w_512 : GCCBuiltin<"__builtin_ia32_ptestnmw512">, - Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, - llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_fpclass_pd_128 : GCCBuiltin<"__builtin_ia32_fpclasspd128_mask">, @@ -1634,6 +1607,25 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrArgMemOnly]>; } +// BITALG bits shuffle +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_vpshufbitqmb_128 : + GCCBuiltin<"__builtin_ia32_vpshufbitqmb128_mask">, + Intrinsic<[llvm_i16_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_vpshufbitqmb_256 : + GCCBuiltin<"__builtin_ia32_vpshufbitqmb256_mask">, + Intrinsic<[llvm_i32_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_vpshufbitqmb_512 : + GCCBuiltin<"__builtin_ia32_vpshufbitqmb512_mask">, + Intrinsic<[llvm_i64_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; +} + //===----------------------------------------------------------------------===// // AVX2 @@ -1678,12 +1670,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, - llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty], [IntrNoMem, Commutative]>; def int_x86_avx2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw256">, Intrinsic<[llvm_v4i64_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem, Commutative]>; @@ -1841,88 +1827,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i32_ty], [IntrNoMem]>; } -// Absolute value ops -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pabs_b : GCCBuiltin<"__builtin_ia32_pabsb256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty], [IntrNoMem]>; - def int_x86_avx2_pabs_w : GCCBuiltin<"__builtin_ia32_pabsw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty], [IntrNoMem]>; - def int_x86_avx2_pabs_d : GCCBuiltin<"__builtin_ia32_pabsd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_b_128 : - GCCBuiltin<"__builtin_ia32_pabsb128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_b_256 : - GCCBuiltin<"__builtin_ia32_pabsb256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_b_512 : - GCCBuiltin<"__builtin_ia32_pabsb512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_d_128 : - GCCBuiltin<"__builtin_ia32_pabsd128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_d_256 : - GCCBuiltin<"__builtin_ia32_pabsd256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_d_512 : - GCCBuiltin<"__builtin_ia32_pabsd512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_q_128 : - GCCBuiltin<"__builtin_ia32_pabsq128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_q_256 : - GCCBuiltin<"__builtin_ia32_pabsq256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_q_512 : - GCCBuiltin<"__builtin_ia32_pabsq512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_w_128 : - GCCBuiltin<"__builtin_ia32_pabsw128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_w_256 : - GCCBuiltin<"__builtin_ia32_pabsw256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pabs_w_512 : - GCCBuiltin<"__builtin_ia32_pabsw512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; -} - // Horizontal arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_phadd_w : GCCBuiltin<"__builtin_ia32_phaddw256">, @@ -1984,65 +1888,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i8_ty], [IntrNoMem]>; } -// Vector load with broadcast -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_pbroadcast_b_gpr_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastb128_gpr_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_b_gpr_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastb256_gpr_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_b_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastb512_gpr_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pbroadcast_w_gpr_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastw128_gpr_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_w_gpr_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastw256_gpr_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_w_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastw512_gpr_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pbroadcast_d_gpr_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastd128_gpr_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_d_gpr_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastd256_gpr_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_d_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastd512_gpr_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pbroadcast_q_gpr_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastq128_gpr_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_q_gpr_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastq256_gpr_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pbroadcast_q_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512_gpr_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_pbroadcast_q_mem_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512_mem_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; -} // Vector permutation let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". @@ -2052,9 +1897,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_permps : GCCBuiltin<"__builtin_ia32_permvarsf256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx2_vperm2i128 : GCCBuiltin<"__builtin_ia32_permti256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; } // Conditional load ops @@ -2346,11 +2188,19 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FMA3 and FMA4 let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_fma_vfmadd_ss : GCCBuiltin<"__builtin_ia32_vfmaddss">, + def int_x86_fma_vfmadd_ss : GCCBuiltin<"__builtin_ia32_vfmaddss3">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], + [IntrNoMem]>; + def int_x86_fma_vfmadd_sd : GCCBuiltin<"__builtin_ia32_vfmaddsd3">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], + [IntrNoMem]>; + def int_x86_fma4_vfmadd_ss : GCCBuiltin<"__builtin_ia32_vfmaddss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmadd_sd : GCCBuiltin<"__builtin_ia32_vfmaddsd">, + def int_x86_fma4_vfmadd_sd : GCCBuiltin<"__builtin_ia32_vfmaddsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; @@ -2371,75 +2221,75 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_ss : GCCBuiltin<"__builtin_ia32_vfmsubss">, + def int_x86_fma_vfmsub_ss : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_sd : GCCBuiltin<"__builtin_ia32_vfmsubsd">, + def int_x86_fma_vfmsub_sd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_ps : GCCBuiltin<"__builtin_ia32_vfmsubps">, + def int_x86_fma_vfmsub_ps : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_pd : GCCBuiltin<"__builtin_ia32_vfmsubpd">, + def int_x86_fma_vfmsub_pd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfmsubps256">, + def int_x86_fma_vfmsub_ps_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfmsubpd256">, + def int_x86_fma_vfmsub_pd_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_ss : GCCBuiltin<"__builtin_ia32_vfnmaddss">, + def int_x86_fma_vfnmadd_ss : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_sd : GCCBuiltin<"__builtin_ia32_vfnmaddsd">, + def int_x86_fma_vfnmadd_sd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_ps : GCCBuiltin<"__builtin_ia32_vfnmaddps">, + def int_x86_fma_vfnmadd_ps : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_pd : GCCBuiltin<"__builtin_ia32_vfnmaddpd">, + def int_x86_fma_vfnmadd_pd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmaddps256">, + def int_x86_fma_vfnmadd_ps_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmadd_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmaddpd256">, + def int_x86_fma_vfnmadd_pd_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_ss : GCCBuiltin<"__builtin_ia32_vfnmsubss">, + def int_x86_fma_vfnmsub_ss : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_sd : GCCBuiltin<"__builtin_ia32_vfnmsubsd">, + def int_x86_fma_vfnmsub_sd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_ps : GCCBuiltin<"__builtin_ia32_vfnmsubps">, + def int_x86_fma_vfnmsub_ps : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_pd : GCCBuiltin<"__builtin_ia32_vfnmsubpd">, + def int_x86_fma_vfnmsub_pd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_ps_256 : GCCBuiltin<"__builtin_ia32_vfnmsubps256">, + def int_x86_fma_vfnmsub_ps_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_fma_vfnmsub_pd_256 : GCCBuiltin<"__builtin_ia32_vfnmsubpd256">, + def int_x86_fma_vfnmsub_pd_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; @@ -2461,21 +2311,19 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_ps : GCCBuiltin<"__builtin_ia32_vfmsubaddps">, + def int_x86_fma_vfmsubadd_ps : // TODO: remove this intrinsic Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_pd : GCCBuiltin<"__builtin_ia32_vfmsubaddpd">, + def int_x86_fma_vfmsubadd_pd : // TODO: remove this intrinsic Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_ps_256 : - GCCBuiltin<"__builtin_ia32_vfmsubaddps256">, + def int_x86_fma_vfmsubadd_ps_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_fma_vfmsubadd_pd_256 : - GCCBuiltin<"__builtin_ia32_vfmsubaddpd256">, + def int_x86_fma_vfmsubadd_pd_256 : // TODO: remove this intrinsic Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty], [IntrNoMem]>; @@ -2987,6 +2835,109 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } +// VNNI +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_vpdpbusd_128 : + GCCBuiltin<"__builtin_ia32_vpdpbusd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusd_128 : + GCCBuiltin<"__builtin_ia32_vpdpbusd128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpbusd_256 : + GCCBuiltin<"__builtin_ia32_vpdpbusd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusd_256 : + GCCBuiltin<"__builtin_ia32_vpdpbusd256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpbusd_512 : + GCCBuiltin<"__builtin_ia32_vpdpbusd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusd_512 : + GCCBuiltin<"__builtin_ia32_vpdpbusd512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpdpbusds_128 : + GCCBuiltin<"__builtin_ia32_vpdpbusds128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusds_128 : + GCCBuiltin<"__builtin_ia32_vpdpbusds128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpbusds_256 : + GCCBuiltin<"__builtin_ia32_vpdpbusds256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusds_256 : + GCCBuiltin<"__builtin_ia32_vpdpbusds256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpbusds_512 : + GCCBuiltin<"__builtin_ia32_vpdpbusds512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpbusds_512 : + GCCBuiltin<"__builtin_ia32_vpdpbusds512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpdpwssd_128 : + GCCBuiltin<"__builtin_ia32_vpdpwssd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssd_128 : + GCCBuiltin<"__builtin_ia32_vpdpwssd128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpwssd_256 : + GCCBuiltin<"__builtin_ia32_vpdpwssd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssd_256 : + GCCBuiltin<"__builtin_ia32_vpdpwssd256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpwssd_512 : + GCCBuiltin<"__builtin_ia32_vpdpwssd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssd_512 : + GCCBuiltin<"__builtin_ia32_vpdpwssd512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpdpwssds_128 : + GCCBuiltin<"__builtin_ia32_vpdpwssds128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssds_128 : + GCCBuiltin<"__builtin_ia32_vpdpwssds128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpwssds_256 : + GCCBuiltin<"__builtin_ia32_vpdpwssds256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssds_256 : + GCCBuiltin<"__builtin_ia32_vpdpwssds256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpdpwssds_512 : + GCCBuiltin<"__builtin_ia32_vpdpwssds512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpdpwssds_512 : + GCCBuiltin<"__builtin_ia32_vpdpwssds512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; +} + //===----------------------------------------------------------------------===// // XOP @@ -3648,10 +3599,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". } //===----------------------------------------------------------------------===// -// CLFLUSHOPT +// CLFLUSHOPT and CLWB let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_clflushopt : GCCBuiltin<"__builtin_ia32_clflushopt">, Intrinsic<[], [llvm_ptr_ty], []>; + + def int_x86_clwb : GCCBuiltin<"__builtin_ia32_clwb">, + Intrinsic<[], [llvm_ptr_ty], []>; } //===----------------------------------------------------------------------===// @@ -3767,32 +3721,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Mask instructions // 16-bit mask - def int_x86_avx512_kand_w : GCCBuiltin<"__builtin_ia32_kandhi">, + def int_x86_avx512_kand_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_kandn_w : GCCBuiltin<"__builtin_ia32_kandnhi">, + def int_x86_avx512_kandn_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_knot_w : GCCBuiltin<"__builtin_ia32_knothi">, + def int_x86_avx512_knot_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_kor_w : GCCBuiltin<"__builtin_ia32_korhi">, + def int_x86_avx512_kor_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_kxor_w : GCCBuiltin<"__builtin_ia32_kxorhi">, + def int_x86_avx512_kxor_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_kxnor_w : GCCBuiltin<"__builtin_ia32_kxnorhi">, + def int_x86_avx512_kxnor_w : // TODO: remove this intrinsic Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_kunpck_bw : GCCBuiltin<"__builtin_ia32_kunpckhi">, - Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_kunpck_wd : GCCBuiltin<"__builtin_ia32_kunpcksi">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_kunpck_dq : GCCBuiltin<"__builtin_ia32_kunpckdi">, - Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], - [IntrNoMem]>; def int_x86_avx512_kortestz_w : GCCBuiltin<"__builtin_ia32_kortestzhi">, Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; @@ -4407,99 +4352,13 @@ def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mas // Vector load with broadcast let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + // TODO: Remove the broadcast intrinsics with no gcc builtin and autoupgrade def int_x86_avx512_vbroadcast_ss_512 : - GCCBuiltin<"__builtin_ia32_vbroadcastss512">, Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_vbroadcast_sd_512 : - GCCBuiltin<"__builtin_ia32_vbroadcastsd512">, Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_mask_broadcastf32x2_256 : - GCCBuiltin<"__builtin_ia32_broadcastf32x2_256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf32x2_512 : - GCCBuiltin<"__builtin_ia32_broadcastf32x2_512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x2_128 : - GCCBuiltin<"__builtin_ia32_broadcasti32x2_128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x2_256 : - GCCBuiltin<"__builtin_ia32_broadcasti32x2_256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x2_512 : - GCCBuiltin<"__builtin_ia32_broadcasti32x2_512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf32x4_256 : - GCCBuiltin<"__builtin_ia32_broadcastf32x4_256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf32x4_512 : - GCCBuiltin<"__builtin_ia32_broadcastf32x4_512">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf32x8_512 : - GCCBuiltin<"__builtin_ia32_broadcastf32x8_512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v8f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf64x2_256 : - GCCBuiltin<"__builtin_ia32_broadcastf64x2_256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v2f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf64x2_512 : - GCCBuiltin<"__builtin_ia32_broadcastf64x2_512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v2f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcastf64x4_512 : - GCCBuiltin<"__builtin_ia32_broadcastf64x4_512">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v4f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x4_256 : - GCCBuiltin<"__builtin_ia32_broadcasti32x4_256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x4_512 : - GCCBuiltin<"__builtin_ia32_broadcasti32x4_512">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti32x8_512 : - GCCBuiltin<"__builtin_ia32_broadcasti32x8_512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v8i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti64x2_256 : - GCCBuiltin<"__builtin_ia32_broadcasti64x2_256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti64x2_512 : - GCCBuiltin<"__builtin_ia32_broadcasti64x2_512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_broadcasti64x4_512 : - GCCBuiltin<"__builtin_ia32_broadcasti64x4_512">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v4i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_broadcastmw_512 : GCCBuiltin<"__builtin_ia32_broadcastmw512">, Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; @@ -5013,24 +4872,6 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_pmulh_w_256 : GCCBuiltin<"__builtin_ia32_pmulhw256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, - llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, - llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_b_128 : GCCBuiltin<"__builtin_ia32_pavgb128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, - llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_b_256 : GCCBuiltin<"__builtin_ia32_pavgb256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, - llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_w_128 : GCCBuiltin<"__builtin_ia32_pavgw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, - llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pavg_w_256 : GCCBuiltin<"__builtin_ia32_pavgw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, - llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaddw_d_128 : GCCBuiltin<"__builtin_ia32_pmaddwd128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -5524,6 +5365,56 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_b_512 : + GCCBuiltin<"__builtin_ia32_compressqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_w_512 : + GCCBuiltin<"__builtin_ia32_compresshi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_b_256 : + GCCBuiltin<"__builtin_ia32_compressqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_w_256 : + GCCBuiltin<"__builtin_ia32_compresshi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_b_128 : + GCCBuiltin<"__builtin_ia32_compressqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress_w_128 : + GCCBuiltin<"__builtin_ia32_compresshi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_compress_store_b_512 : + GCCBuiltin<"__builtin_ia32_compressstoreqi512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v64i8_ty, + llvm_i64_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_store_w_512 : + GCCBuiltin<"__builtin_ia32_compressstorehi512_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_store_b_256 : + GCCBuiltin<"__builtin_ia32_compressstoreqi256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v32i8_ty, + llvm_i32_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_store_w_256 : + GCCBuiltin<"__builtin_ia32_compressstorehi256_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_store_b_128 : + GCCBuiltin<"__builtin_ia32_compressstoreqi128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v16i8_ty, + llvm_i16_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_compress_store_w_128 : + GCCBuiltin<"__builtin_ia32_compressstorehi128_mask">, + Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrArgMemOnly]>; + // expand def int_x86_avx512_mask_expand_ps_512 : GCCBuiltin<"__builtin_ia32_expandsf512_mask">, @@ -5625,6 +5516,304 @@ let TargetPrefix = "x86" in { Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_b_512 : + GCCBuiltin<"__builtin_ia32_expandqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_w_512 : + GCCBuiltin<"__builtin_ia32_expandhi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_b_256 : + GCCBuiltin<"__builtin_ia32_expandqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_w_256 : + GCCBuiltin<"__builtin_ia32_expandhi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_b_128 : + GCCBuiltin<"__builtin_ia32_expandqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_expand_w_128 : + GCCBuiltin<"__builtin_ia32_expandhi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_expand_load_b_512 : + GCCBuiltin<"__builtin_ia32_expandloadqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_ptr_ty, llvm_v64i8_ty, + llvm_i64_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_load_w_512 : + GCCBuiltin<"__builtin_ia32_expandloadhi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_ptr_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_load_b_256 : + GCCBuiltin<"__builtin_ia32_expandloadqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_v32i8_ty, + llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_load_w_256 : + GCCBuiltin<"__builtin_ia32_expandloadhi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_ptr_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_load_b_128 : + GCCBuiltin<"__builtin_ia32_expandloadqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_v16i8_ty, + llvm_i16_ty], [IntrReadMem, IntrArgMemOnly]>; + def int_x86_avx512_mask_expand_load_w_128 : + GCCBuiltin<"__builtin_ia32_expandloadhi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; +} + +// VBMI2 Concat & Shift +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_vpshld_q_512 : + GCCBuiltin<"__builtin_ia32_vpshldq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_q_256 : + GCCBuiltin<"__builtin_ia32_vpshldq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_q_128 : + GCCBuiltin<"__builtin_ia32_vpshldq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshld_d_512 : + GCCBuiltin<"__builtin_ia32_vpshldd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_d_256 : + GCCBuiltin<"__builtin_ia32_vpshldd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_d_128 : + GCCBuiltin<"__builtin_ia32_vpshldd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshld_w_512 : + GCCBuiltin<"__builtin_ia32_vpshldw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_w_256 : + GCCBuiltin<"__builtin_ia32_vpshldw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshld_w_128 : + GCCBuiltin<"__builtin_ia32_vpshldw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrd_q_512 : + GCCBuiltin<"__builtin_ia32_vpshrdq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_q_256 : + GCCBuiltin<"__builtin_ia32_vpshrdq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_q_128 : + GCCBuiltin<"__builtin_ia32_vpshrdq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrd_d_512 : + GCCBuiltin<"__builtin_ia32_vpshrdd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_d_256 : + GCCBuiltin<"__builtin_ia32_vpshrdd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_d_128 : + GCCBuiltin<"__builtin_ia32_vpshrdd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrd_w_512 : + GCCBuiltin<"__builtin_ia32_vpshrdw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_w_256 : + GCCBuiltin<"__builtin_ia32_vpshrdw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrd_w_128 : + GCCBuiltin<"__builtin_ia32_vpshrdw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshldv_w_128 : + GCCBuiltin<"__builtin_ia32_vpshldvw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_w_128 : + GCCBuiltin<"__builtin_ia32_vpshldvw128_maskz">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_w_256 : + GCCBuiltin<"__builtin_ia32_vpshldvw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_w_256 : + GCCBuiltin<"__builtin_ia32_vpshldvw256_maskz">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_w_512 : + GCCBuiltin<"__builtin_ia32_vpshldvw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_w_512 : + GCCBuiltin<"__builtin_ia32_vpshldvw512_maskz">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshldv_q_128 : + GCCBuiltin<"__builtin_ia32_vpshldvq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_q_128 : + GCCBuiltin<"__builtin_ia32_vpshldvq128_maskz">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_q_256 : + GCCBuiltin<"__builtin_ia32_vpshldvq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_q_256 : + GCCBuiltin<"__builtin_ia32_vpshldvq256_maskz">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_q_512 : + GCCBuiltin<"__builtin_ia32_vpshldvq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_q_512 : + GCCBuiltin<"__builtin_ia32_vpshldvq512_maskz">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshldv_d_128 : + GCCBuiltin<"__builtin_ia32_vpshldvd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_d_128 : + GCCBuiltin<"__builtin_ia32_vpshldvd128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_d_256 : + GCCBuiltin<"__builtin_ia32_vpshldvd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_d_256 : + GCCBuiltin<"__builtin_ia32_vpshldvd256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshldv_d_512 : + GCCBuiltin<"__builtin_ia32_vpshldvd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshldv_d_512 : + GCCBuiltin<"__builtin_ia32_vpshldvd512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrdv_w_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_w_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvw128_maskz">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_w_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_w_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvw256_maskz">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_w_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_w_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvw512_maskz">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrdv_q_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_q_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvq128_maskz">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_q_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_q_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvq256_maskz">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_q_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_q_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvq512_maskz">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vpshrdv_d_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_d_128 : + GCCBuiltin<"__builtin_ia32_vpshrdvd128_maskz">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_d_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_d_256 : + GCCBuiltin<"__builtin_ia32_vpshrdvd256_maskz">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpshrdv_d_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpshrdv_d_512 : + GCCBuiltin<"__builtin_ia32_vpshrdvd512_maskz">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; } // truncate diff --git a/contrib/llvm/include/llvm/IR/LLVMContext.h b/contrib/llvm/include/llvm/IR/LLVMContext.h index 4cb77701f762..a95634d32c21 100644 --- a/contrib/llvm/include/llvm/IR/LLVMContext.h +++ b/contrib/llvm/include/llvm/IR/LLVMContext.h @@ -16,6 +16,7 @@ #define LLVM_IR_LLVMCONTEXT_H #include "llvm-c/Types.h" +#include "llvm/IR/DiagnosticHandler.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Options.h" #include <cstdint> @@ -99,6 +100,8 @@ public: MD_section_prefix = 20, // "section_prefix" MD_absolute_symbol = 21, // "absolute_symbol" MD_associated = 22, // "associated" + MD_callees = 23, // "callees" + MD_irr_loop = 24, // "irr_loop" }; /// Known operand bundle tag IDs, which always have the same value. All @@ -167,11 +170,6 @@ public: using InlineAsmDiagHandlerTy = void (*)(const SMDiagnostic&, void *Context, unsigned LocCookie); - /// Defines the type of a diagnostic handler. - /// \see LLVMContext::setDiagnosticHandler. - /// \see LLVMContext::diagnose. - using DiagnosticHandlerTy = void (*)(const DiagnosticInfo &DI, void *Context); - /// Defines the type of a yield callback. /// \see LLVMContext::setYieldCallback. using YieldCallbackTy = void (*)(LLVMContext *Context, void *OpaqueHandle); @@ -194,26 +192,43 @@ public: /// setInlineAsmDiagnosticHandler. void *getInlineAsmDiagnosticContext() const; - /// setDiagnosticHandler - This method sets a handler that is invoked - /// when the backend needs to report anything to the user. The first - /// argument is a function pointer and the second is a context pointer that - /// gets passed into the DiagHandler. The third argument should be set to + /// setDiagnosticHandlerCallBack - This method sets a handler call back + /// that is invoked when the backend needs to report anything to the user. + /// The first argument is a function pointer and the second is a context pointer + /// that gets passed into the DiagHandler. The third argument should be set to /// true if the handler only expects enabled diagnostics. /// /// LLVMContext doesn't take ownership or interpret either of these /// pointers. - void setDiagnosticHandler(DiagnosticHandlerTy DiagHandler, - void *DiagContext = nullptr, + void setDiagnosticHandlerCallBack( + DiagnosticHandler::DiagnosticHandlerTy DiagHandler, + void *DiagContext = nullptr, bool RespectFilters = false); + + /// setDiagnosticHandler - This method sets unique_ptr to object of DiagnosticHandler + /// to provide custom diagnostic handling. The first argument is unique_ptr of object + /// of type DiagnosticHandler or a derived of that. The third argument should be + /// set to true if the handler only expects enabled diagnostics. + /// + /// Ownership of this pointer is moved to LLVMContextImpl. + void setDiagnosticHandler(std::unique_ptr<DiagnosticHandler> &&DH, bool RespectFilters = false); - /// getDiagnosticHandler - Return the diagnostic handler set by - /// setDiagnosticHandler. - DiagnosticHandlerTy getDiagnosticHandler() const; + /// getDiagnosticHandlerCallBack - Return the diagnostic handler call back set by + /// setDiagnosticHandlerCallBack. + DiagnosticHandler::DiagnosticHandlerTy getDiagnosticHandlerCallBack() const; /// getDiagnosticContext - Return the diagnostic context set by /// setDiagnosticContext. void *getDiagnosticContext() const; + /// getDiagHandlerPtr - Returns const raw pointer of DiagnosticHandler set by + /// setDiagnosticHandler. + const DiagnosticHandler *getDiagHandlerPtr() const; + + /// getDiagnosticHandler - transfers owenership of DiagnosticHandler unique_ptr + /// to caller. + std::unique_ptr<DiagnosticHandler> getDiagnosticHandler(); + /// \brief Return if a code hotness metric should be included in optimization /// diagnostics. bool getDiagnosticsHotnessRequested() const; diff --git a/contrib/llvm/include/llvm/IR/LegacyPassManagers.h b/contrib/llvm/include/llvm/IR/LegacyPassManagers.h index b22f9302298d..3dc4a776dba0 100644 --- a/contrib/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/contrib/llvm/include/llvm/IR/LegacyPassManagers.h @@ -279,11 +279,11 @@ private: // when we have multiple instances of the same pass since they'll usually // have the same analysis usage and can share storage. FoldingSet<AUFoldingSetNode> UniqueAnalysisUsages; - + // Allocator used for allocating UAFoldingSetNodes. This handles deletion of // all allocated nodes in one fell swoop. SpecificBumpPtrAllocator<AUFoldingSetNode> AUFoldingSetNodeAllocator; - + // Maps from a pass to it's associated entry in UniqueAnalysisUsages. Does // not own the storage associated with either key or value.. DenseMap<Pass *, AnalysisUsage*> AnUsageMap; diff --git a/contrib/llvm/include/llvm/IR/MDBuilder.h b/contrib/llvm/include/llvm/IR/MDBuilder.h index 899976a87bc7..dff1ca12407f 100644 --- a/contrib/llvm/include/llvm/IR/MDBuilder.h +++ b/contrib/llvm/include/llvm/IR/MDBuilder.h @@ -30,6 +30,7 @@ class Constant; class ConstantAsMetadata; class MDNode; class MDString; +class Metadata; class MDBuilder { LLVMContext &Context; @@ -85,6 +86,14 @@ public: MDNode *createRange(Constant *Lo, Constant *Hi); //===------------------------------------------------------------------===// + // Callees metadata. + //===------------------------------------------------------------------===// + + /// \brief Return metadata indicating the possible callees of indirect + /// calls. + MDNode *createCallees(ArrayRef<Function *> Callees); + + //===------------------------------------------------------------------===// // AA metadata. //===------------------------------------------------------------------===// @@ -141,9 +150,9 @@ public: struct TBAAStructField { uint64_t Offset; uint64_t Size; - MDNode *TBAA; - TBAAStructField(uint64_t Offset, uint64_t Size, MDNode *TBAA) : - Offset(Offset), Size(Size), TBAA(TBAA) {} + MDNode *Type; + TBAAStructField(uint64_t Offset, uint64_t Size, MDNode *Type) : + Offset(Offset), Size(Size), Type(Type) {} }; /// \brief Return metadata for a tbaa.struct node with the given @@ -165,6 +174,23 @@ public: /// base type, access type and offset relative to the base type. MDNode *createTBAAStructTagNode(MDNode *BaseType, MDNode *AccessType, uint64_t Offset, bool IsConstant = false); + + /// \brief Return metadata for a TBAA type node in the TBAA type DAG with the + /// given parent type, size in bytes, type identifier and a list of fields. + MDNode *createTBAATypeNode(MDNode *Parent, uint64_t Size, Metadata *Id, + ArrayRef<TBAAStructField> Fields = + ArrayRef<TBAAStructField>()); + + /// \brief Return metadata for a TBAA access tag with the given base type, + /// final access type, offset of the access relative to the base type, size of + /// the access and flag indicating whether the accessed object can be + /// considered immutable for the purposes of the TBAA analysis. + MDNode *createTBAAAccessTag(MDNode *BaseType, MDNode *AccessType, + uint64_t Offset, uint64_t Size, + bool IsImmutable = false); + + /// \brief Return metadata containing an irreducible loop header weight. + MDNode *createIrrLoopHeaderWeight(uint64_t Weight); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/IR/Metadata.h b/contrib/llvm/include/llvm/IR/Metadata.h index 3462cc02fd27..bc0b87a6c348 100644 --- a/contrib/llvm/include/llvm/IR/Metadata.h +++ b/contrib/llvm/include/llvm/IR/Metadata.h @@ -958,6 +958,9 @@ public: /// \pre No operands (or operands' operands, etc.) have \a isTemporary(). void resolveCycles(); + /// Resolve a unique, unresolved node. + void resolve(); + /// \brief Replace a temporary node with a permanent one. /// /// Try to create a uniqued version of \c N -- in place, if possible -- and @@ -1009,9 +1012,6 @@ protected: private: void handleChangedOperand(void *Ref, Metadata *New); - /// Resolve a unique, unresolved node. - void resolve(); - /// Drop RAUW support, if any. void dropReplaceableUses(); @@ -1188,7 +1188,8 @@ void TempMDNodeDeleter::operator()(MDNode *Node) const { /// particular Metadata subclass. template <class T> class TypedMDOperandIterator - : std::iterator<std::input_iterator_tag, T *, std::ptrdiff_t, void, T *> { + : public std::iterator<std::input_iterator_tag, T *, std::ptrdiff_t, void, + T *> { MDNode::op_iterator I = nullptr; public: @@ -1302,7 +1303,13 @@ public: if (!Use) return; *Use = MD; - Use = nullptr; + + if (*Use) + MetadataTracking::track(*Use); + + Metadata *T = cast<Metadata>(this); + MetadataTracking::untrack(T); + assert(!Use && "Use is still being tracked despite being untracked!"); } }; diff --git a/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h b/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h index 4aa8a0199ab1..dd7a0db83774 100644 --- a/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/contrib/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -148,11 +148,15 @@ public: /// In combined summary, indicate that the global value is live. unsigned Live : 1; + /// Indicates that the linker resolved the symbol to a definition from + /// within the same linkage unit. + unsigned DSOLocal : 1; + /// Convenience Constructors explicit GVFlags(GlobalValue::LinkageTypes Linkage, - bool NotEligibleToImport, bool Live) + bool NotEligibleToImport, bool Live, bool IsLocal) : Linkage(Linkage), NotEligibleToImport(NotEligibleToImport), - Live(Live) {} + Live(Live), DSOLocal(IsLocal) {} }; private: @@ -185,7 +189,10 @@ private: protected: GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector<ValueInfo> Refs) - : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {} + : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) { + assert((K != AliasKind || Refs.empty()) && + "Expect no references for AliasSummary"); + } public: virtual ~GlobalValueSummary() = default; @@ -226,12 +233,21 @@ public: void setLive(bool Live) { Flags.Live = Live; } + void setDSOLocal(bool Local) { Flags.DSOLocal = Local; } + + bool isDSOLocal() const { return Flags.DSOLocal; } + /// Flag that this global value cannot be imported. void setNotEligibleToImport() { Flags.NotEligibleToImport = true; } /// Return the list of values referenced by this global value definition. ArrayRef<ValueInfo> refs() const { return RefEdgeList; } + /// If this is an alias summary, returns the summary of the aliased object (a + /// global variable or function), otherwise returns itself. + GlobalValueSummary *getBaseObject(); + const GlobalValueSummary *getBaseObject() const; + friend class ModuleSummaryIndex; friend void computeDeadSymbols(class ModuleSummaryIndex &, const DenseSet<GlobalValue::GUID> &); @@ -240,10 +256,14 @@ public: /// \brief Alias summary information. class AliasSummary : public GlobalValueSummary { GlobalValueSummary *AliaseeSummary; + // AliaseeGUID is only set and accessed when we are building a combined index + // via the BitcodeReader. + GlobalValue::GUID AliaseeGUID; public: - AliasSummary(GVFlags Flags, std::vector<ValueInfo> Refs) - : GlobalValueSummary(AliasKind, Flags, std::move(Refs)) {} + AliasSummary(GVFlags Flags) + : GlobalValueSummary(AliasKind, Flags, ArrayRef<ValueInfo>{}), + AliaseeSummary(nullptr), AliaseeGUID(0) {} /// Check if this is an alias summary. static bool classof(const GlobalValueSummary *GVS) { @@ -251,6 +271,7 @@ public: } void setAliasee(GlobalValueSummary *Aliasee) { AliaseeSummary = Aliasee; } + void setAliaseeGUID(GlobalValue::GUID GUID) { AliaseeGUID = GUID; } const GlobalValueSummary &getAliasee() const { assert(AliaseeSummary && "Unexpected missing aliasee summary"); @@ -261,8 +282,24 @@ public: return const_cast<GlobalValueSummary &>( static_cast<const AliasSummary *>(this)->getAliasee()); } + const GlobalValue::GUID &getAliaseeGUID() const { + assert(AliaseeGUID && "Unexpected missing aliasee GUID"); + return AliaseeGUID; + } }; +const inline GlobalValueSummary *GlobalValueSummary::getBaseObject() const { + if (auto *AS = dyn_cast<AliasSummary>(this)) + return &AS->getAliasee(); + return this; +} + +inline GlobalValueSummary *GlobalValueSummary::getBaseObject() { + if (auto *AS = dyn_cast<AliasSummary>(this)) + return &AS->getAliasee(); + return this; +} + /// \brief Function summary information to aid decisions and implementation of /// importing. class FunctionSummary : public GlobalValueSummary { @@ -273,7 +310,7 @@ public: /// An "identifier" for a virtual function. This contains the type identifier /// represented as a GUID and the offset from the address point to the virtual /// function pointer, where "address point" is as defined in the Itanium ABI: - /// https://mentorembedded.github.io/cxx-abi/abi.html#vtable-general + /// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vtable-general struct VFuncId { GlobalValue::GUID GUID; uint64_t Offset; @@ -287,11 +324,24 @@ public: std::vector<uint64_t> Args; }; + /// Function attribute flags. Used to track if a function accesses memory, + /// recurses or aliases. + struct FFlags { + unsigned ReadNone : 1; + unsigned ReadOnly : 1; + unsigned NoRecurse : 1; + unsigned ReturnDoesNotAlias : 1; + }; + private: /// Number of instructions (ignoring debug instructions, e.g.) computed /// during the initial compile step when the summary index is first built. unsigned InstCount; + /// Function attribute flags. Used to track if a function accesses memory, + /// recurses or aliases. + FFlags FunFlags; + /// List of <CalleeValueInfo, CalleeInfo> call edge pairs from this function. std::vector<EdgeTy> CallGraphEdgeList; @@ -317,15 +367,16 @@ private: std::unique_ptr<TypeIdInfo> TIdInfo; public: - FunctionSummary(GVFlags Flags, unsigned NumInsts, std::vector<ValueInfo> Refs, - std::vector<EdgeTy> CGEdges, + FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags, + std::vector<ValueInfo> Refs, std::vector<EdgeTy> CGEdges, std::vector<GlobalValue::GUID> TypeTests, std::vector<VFuncId> TypeTestAssumeVCalls, std::vector<VFuncId> TypeCheckedLoadVCalls, std::vector<ConstVCall> TypeTestAssumeConstVCalls, std::vector<ConstVCall> TypeCheckedLoadConstVCalls) : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)), - InstCount(NumInsts), CallGraphEdgeList(std::move(CGEdges)) { + InstCount(NumInsts), FunFlags(FunFlags), + CallGraphEdgeList(std::move(CGEdges)) { if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() || !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() || !TypeCheckedLoadConstVCalls.empty()) @@ -341,6 +392,9 @@ public: return GVS->getSummaryKind() == FunctionKind; } + /// Get function attribute flags. + FFlags &fflags() { return FunFlags; } + /// Get the instruction count recorded for this function. unsigned instCount() const { return InstCount; } @@ -470,6 +524,16 @@ struct TypeTestResolution { /// range [1,256], this number will be 8. This helps generate the most compact /// instruction sequences. unsigned SizeM1BitWidth = 0; + + // The following fields are only used if the target does not support the use + // of absolute symbols to store constants. Their meanings are the same as the + // corresponding fields in LowerTypeTestsModule::TypeIdLowering in + // LowerTypeTests.cpp. + + uint64_t AlignLog2 = 0; + uint64_t SizeM1 = 0; + uint8_t BitMask = 0; + uint64_t InlineBits = 0; }; struct WholeProgramDevirtResolution { @@ -493,6 +557,12 @@ struct WholeProgramDevirtResolution { /// - UniqueRetVal: the return value associated with the unique vtable (0 or /// 1). uint64_t Info = 0; + + // The following fields are only used if the target does not support the use + // of absolute symbols to store constants. + + uint32_t Byte = 0; + uint32_t Bit = 0; }; /// Resolutions for calls with all constant integer arguments (excluding the @@ -697,7 +767,8 @@ public: static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash) { SmallString<256> NewName(Name); NewName += ".llvm."; - NewName += utohexstr(ModHash[0]); // Take the first 32 bits + NewName += utostr((uint64_t(ModHash[0]) << 32) | + ModHash[1]); // Take the first 64 bits return NewName.str(); } diff --git a/contrib/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/contrib/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h index 7f6cb5bee5a6..4687f2d53e7e 100644 --- a/contrib/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/contrib/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -30,6 +30,10 @@ template <> struct MappingTraits<TypeTestResolution> { static void mapping(IO &io, TypeTestResolution &res) { io.mapOptional("Kind", res.TheKind); io.mapOptional("SizeM1BitWidth", res.SizeM1BitWidth); + io.mapOptional("AlignLog2", res.AlignLog2); + io.mapOptional("SizeM1", res.SizeM1); + io.mapOptional("BitMask", res.BitMask); + io.mapOptional("InlineBits", res.InlineBits); } }; @@ -51,6 +55,8 @@ template <> struct MappingTraits<WholeProgramDevirtResolution::ByArg> { static void mapping(IO &io, WholeProgramDevirtResolution::ByArg &res) { io.mapOptional("Kind", res.TheKind); io.mapOptional("Info", res.Info); + io.mapOptional("Byte", res.Byte); + io.mapOptional("Bit", res.Bit); } }; @@ -129,7 +135,7 @@ template <> struct MappingTraits<TypeIdSummary> { struct FunctionSummaryYaml { unsigned Linkage; - bool NotEligibleToImport, Live; + bool NotEligibleToImport, Live, IsLocal; std::vector<uint64_t> TypeTests; std::vector<FunctionSummary::VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls; @@ -171,6 +177,7 @@ template <> struct MappingTraits<FunctionSummaryYaml> { io.mapOptional("Linkage", summary.Linkage); io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport); io.mapOptional("Live", summary.Live); + io.mapOptional("Local", summary.IsLocal); io.mapOptional("TypeTests", summary.TypeTests); io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls); io.mapOptional("TypeCheckedLoadVCalls", summary.TypeCheckedLoadVCalls); @@ -205,9 +212,10 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { Elem.SummaryList.push_back(llvm::make_unique<FunctionSummary>( GlobalValueSummary::GVFlags( static_cast<GlobalValue::LinkageTypes>(FSum.Linkage), - FSum.NotEligibleToImport, FSum.Live), - 0, ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{}, - std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), + FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal), + 0, FunctionSummary::FFlags{}, ArrayRef<ValueInfo>{}, + ArrayRef<FunctionSummary::EdgeTy>{}, std::move(FSum.TypeTests), + std::move(FSum.TypeTestAssumeVCalls), std::move(FSum.TypeCheckedLoadVCalls), std::move(FSum.TypeTestAssumeConstVCalls), std::move(FSum.TypeCheckedLoadConstVCalls))); @@ -221,7 +229,8 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { FSums.push_back(FunctionSummaryYaml{ FSum->flags().Linkage, static_cast<bool>(FSum->flags().NotEligibleToImport), - static_cast<bool>(FSum->flags().Live), FSum->type_tests(), + static_cast<bool>(FSum->flags().Live), + static_cast<bool>(FSum->flags().DSOLocal), FSum->type_tests(), FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(), FSum->type_test_assume_const_vcalls(), FSum->type_checked_load_const_vcalls()}); diff --git a/contrib/llvm/include/llvm/IR/Operator.h b/contrib/llvm/include/llvm/IR/Operator.h index 9df6bfc54cd4..01746e4b6a29 100644 --- a/contrib/llvm/include/llvm/IR/Operator.h +++ b/contrib/llvm/include/llvm/IR/Operator.h @@ -61,9 +61,9 @@ public: } }; -/// Utility class for integer arithmetic operators which may exhibit overflow - -/// Add, Sub, and Mul. It does not include SDiv, despite that operator having -/// the potential for overflow. +/// Utility class for integer operators which may exhibit overflow - Add, Sub, +/// Mul, and Shl. It does not include SDiv, despite that operator having the +/// potential for overflow. class OverflowingBinaryOperator : public Operator { public: enum { @@ -163,52 +163,61 @@ private: unsigned Flags = 0; - FastMathFlags(unsigned F) : Flags(F) { } + FastMathFlags(unsigned F) { + // If all 7 bits are set, turn this into -1. If the number of bits grows, + // this must be updated. This is intended to provide some forward binary + // compatibility insurance for the meaning of 'fast' in case bits are added. + if (F == 0x7F) Flags = ~0U; + else Flags = F; + } public: - /// This is how the bits are used in Value::SubclassOptionalData so they - /// should fit there too. + // This is how the bits are used in Value::SubclassOptionalData so they + // should fit there too. + // WARNING: We're out of space. SubclassOptionalData only has 7 bits. New + // functionality will require a change in how this information is stored. enum { - UnsafeAlgebra = (1 << 0), + AllowReassoc = (1 << 0), NoNaNs = (1 << 1), NoInfs = (1 << 2), NoSignedZeros = (1 << 3), AllowReciprocal = (1 << 4), - AllowContract = (1 << 5) + AllowContract = (1 << 5), + ApproxFunc = (1 << 6) }; FastMathFlags() = default; - /// Whether any flag is set bool any() const { return Flags != 0; } + bool none() const { return Flags == 0; } + bool all() const { return Flags == ~0U; } - /// Set all the flags to false void clear() { Flags = 0; } + void set() { Flags = ~0U; } /// Flag queries + bool allowReassoc() const { return 0 != (Flags & AllowReassoc); } bool noNaNs() const { return 0 != (Flags & NoNaNs); } bool noInfs() const { return 0 != (Flags & NoInfs); } bool noSignedZeros() const { return 0 != (Flags & NoSignedZeros); } bool allowReciprocal() const { return 0 != (Flags & AllowReciprocal); } - bool allowContract() const { return 0 != (Flags & AllowContract); } - bool unsafeAlgebra() const { return 0 != (Flags & UnsafeAlgebra); } + bool allowContract() const { return 0 != (Flags & AllowContract); } + bool approxFunc() const { return 0 != (Flags & ApproxFunc); } + /// 'Fast' means all bits are set. + bool isFast() const { return all(); } /// Flag setters + void setAllowReassoc() { Flags |= AllowReassoc; } void setNoNaNs() { Flags |= NoNaNs; } void setNoInfs() { Flags |= NoInfs; } void setNoSignedZeros() { Flags |= NoSignedZeros; } void setAllowReciprocal() { Flags |= AllowReciprocal; } + // TODO: Change the other set* functions to take a parameter? void setAllowContract(bool B) { Flags = (Flags & ~AllowContract) | B * AllowContract; } - void setUnsafeAlgebra() { - Flags |= UnsafeAlgebra; - setNoNaNs(); - setNoInfs(); - setNoSignedZeros(); - setAllowReciprocal(); - setAllowContract(true); - } + void setApproxFunc() { Flags |= ApproxFunc; } + void setFast() { set(); } void operator&=(const FastMathFlags &OtherFlags) { Flags &= OtherFlags.Flags; @@ -221,18 +230,21 @@ class FPMathOperator : public Operator { private: friend class Instruction; - void setHasUnsafeAlgebra(bool B) { + /// 'Fast' means all bits are set. + void setFast(bool B) { + setHasAllowReassoc(B); + setHasNoNaNs(B); + setHasNoInfs(B); + setHasNoSignedZeros(B); + setHasAllowReciprocal(B); + setHasAllowContract(B); + setHasApproxFunc(B); + } + + void setHasAllowReassoc(bool B) { SubclassOptionalData = - (SubclassOptionalData & ~FastMathFlags::UnsafeAlgebra) | - (B * FastMathFlags::UnsafeAlgebra); - - // Unsafe algebra implies all the others - if (B) { - setHasNoNaNs(true); - setHasNoInfs(true); - setHasNoSignedZeros(true); - setHasAllowReciprocal(true); - } + (SubclassOptionalData & ~FastMathFlags::AllowReassoc) | + (B * FastMathFlags::AllowReassoc); } void setHasNoNaNs(bool B) { @@ -265,6 +277,12 @@ private: (B * FastMathFlags::AllowContract); } + void setHasApproxFunc(bool B) { + SubclassOptionalData = + (SubclassOptionalData & ~FastMathFlags::ApproxFunc) | + (B * FastMathFlags::ApproxFunc); + } + /// Convenience function for setting multiple fast-math flags. /// FMF is a mask of the bits to set. void setFastMathFlags(FastMathFlags FMF) { @@ -278,42 +296,53 @@ private: } public: - /// Test whether this operation is permitted to be - /// algebraically transformed, aka the 'A' fast-math property. - bool hasUnsafeAlgebra() const { - return (SubclassOptionalData & FastMathFlags::UnsafeAlgebra) != 0; + /// Test if this operation allows all non-strict floating-point transforms. + bool isFast() const { + return ((SubclassOptionalData & FastMathFlags::AllowReassoc) != 0 && + (SubclassOptionalData & FastMathFlags::NoNaNs) != 0 && + (SubclassOptionalData & FastMathFlags::NoInfs) != 0 && + (SubclassOptionalData & FastMathFlags::NoSignedZeros) != 0 && + (SubclassOptionalData & FastMathFlags::AllowReciprocal) != 0 && + (SubclassOptionalData & FastMathFlags::AllowContract) != 0 && + (SubclassOptionalData & FastMathFlags::ApproxFunc) != 0); + } + + /// Test if this operation may be simplified with reassociative transforms. + bool hasAllowReassoc() const { + return (SubclassOptionalData & FastMathFlags::AllowReassoc) != 0; } - /// Test whether this operation's arguments and results are to be - /// treated as non-NaN, aka the 'N' fast-math property. + /// Test if this operation's arguments and results are assumed not-NaN. bool hasNoNaNs() const { return (SubclassOptionalData & FastMathFlags::NoNaNs) != 0; } - /// Test whether this operation's arguments and results are to be - /// treated as NoN-Inf, aka the 'I' fast-math property. + /// Test if this operation's arguments and results are assumed not-infinite. bool hasNoInfs() const { return (SubclassOptionalData & FastMathFlags::NoInfs) != 0; } - /// Test whether this operation can treat the sign of zero - /// as insignificant, aka the 'S' fast-math property. + /// Test if this operation can ignore the sign of zero. bool hasNoSignedZeros() const { return (SubclassOptionalData & FastMathFlags::NoSignedZeros) != 0; } - /// Test whether this operation is permitted to use - /// reciprocal instead of division, aka the 'R' fast-math property. + /// Test if this operation can use reciprocal multiply instead of division. bool hasAllowReciprocal() const { return (SubclassOptionalData & FastMathFlags::AllowReciprocal) != 0; } - /// Test whether this operation is permitted to - /// be floating-point contracted. + /// Test if this operation can be floating-point contracted (FMA). bool hasAllowContract() const { return (SubclassOptionalData & FastMathFlags::AllowContract) != 0; } + /// Test if this operation allows approximations of math library functions or + /// intrinsics. + bool hasApproxFunc() const { + return (SubclassOptionalData & FastMathFlags::ApproxFunc) != 0; + } + /// Convenience function for getting all the fast-math flags FastMathFlags getFastMathFlags() const { return FastMathFlags(SubclassOptionalData); @@ -472,6 +501,12 @@ public: return true; } + unsigned countNonConstantIndices() const { + return count_if(make_range(idx_begin(), idx_end()), [](const Use& use) { + return !isa<ConstantInt>(*use); + }); + } + /// \brief Accumulate the constant address offset of this GEP if possible. /// /// This routine accepts an APInt into which it will accumulate the constant diff --git a/contrib/llvm/include/llvm/IR/OptBisect.h b/contrib/llvm/include/llvm/IR/OptBisect.h index 185a5ac956f5..09e67aa79246 100644 --- a/contrib/llvm/include/llvm/IR/OptBisect.h +++ b/contrib/llvm/include/llvm/IR/OptBisect.h @@ -1,4 +1,4 @@ -//===----------- llvm/IR/OptBisect.h - LLVM Bisect support -------------===// +//===- llvm/IR/OptBisect.h - LLVM Bisect support ----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,11 +15,11 @@ #ifndef LLVM_IR_OPTBISECT_H #define LLVM_IR_OPTBISECT_H +#include "llvm/ADT/StringRef.h" + namespace llvm { class Pass; -class StringRef; -class Twine; /// This class implements a mechanism to disable passes and individual /// optimizations at compile time based on a command line option diff --git a/contrib/llvm/include/llvm/IR/PassManager.h b/contrib/llvm/include/llvm/IR/PassManager.h index 393175675034..4f838a719512 100644 --- a/contrib/llvm/include/llvm/IR/PassManager.h +++ b/contrib/llvm/include/llvm/IR/PassManager.h @@ -470,7 +470,7 @@ public: //IR.getContext().yield(); } - // Invaliadtion was handled after each pass in the above loop for the + // Invalidation was handled after each pass in the above loop for the // current unit of IR. Therefore, the remaining analysis results in the // AnalysisManager are preserved. We mark this with a set so that we don't // need to inspect each one individually. @@ -654,9 +654,9 @@ public: /// This doesn't invalidate, but instead simply deletes, the relevant results. /// It is useful when the IR is being removed and we want to clear out all the /// memory pinned for it. - void clear(IRUnitT &IR) { + void clear(IRUnitT &IR, llvm::StringRef Name) { if (DebugLogging) - dbgs() << "Clearing all analysis results for: " << IR.getName() << "\n"; + dbgs() << "Clearing all analysis results for: " << Name << "\n"; auto ResultsListI = AnalysisResultLists.find(&IR); if (ResultsListI == AnalysisResultLists.end()) diff --git a/contrib/llvm/include/llvm/IR/PatternMatch.h b/contrib/llvm/include/llvm/IR/PatternMatch.h index acb895211644..245d72fbd16e 100644 --- a/contrib/llvm/include/llvm/IR/PatternMatch.h +++ b/contrib/llvm/include/llvm/IR/PatternMatch.h @@ -234,11 +234,35 @@ struct apint_match { return false; } }; +// Either constexpr if or renaming ConstantFP::getValueAPF to +// ConstantFP::getValue is needed to do it via single template +// function for both apint/apfloat. +struct apfloat_match { + const APFloat *&Res; + apfloat_match(const APFloat *&R) : Res(R) {} + template <typename ITy> bool match(ITy *V) { + if (auto *CI = dyn_cast<ConstantFP>(V)) { + Res = &CI->getValueAPF(); + return true; + } + if (V->getType()->isVectorTy()) + if (const auto *C = dyn_cast<Constant>(V)) + if (auto *CI = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) { + Res = &CI->getValueAPF(); + return true; + } + return false; + } +}; /// \brief Match a ConstantInt or splatted ConstantVector, binding the /// specified pointer to the contained APInt. inline apint_match m_APInt(const APInt *&Res) { return Res; } +/// \brief Match a ConstantFP or splatted ConstantVector, binding the +/// specified pointer to the contained APFloat. +inline apfloat_match m_APFloat(const APFloat *&Res) { return Res; } + template <int64_t Val> struct constantint_match { template <typename ITy> bool match(ITy *V) { if (const auto *CI = dyn_cast<ConstantInt>(V)) { @@ -933,6 +957,26 @@ inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { } //===----------------------------------------------------------------------===// +// Matcher for LoadInst classes +// + +template <typename Op_t> struct LoadClass_match { + Op_t Op; + + LoadClass_match(const Op_t &OpMatch) : Op(OpMatch) {} + + template <typename OpTy> bool match(OpTy *V) { + if (auto *LI = dyn_cast<LoadInst>(V)) + return Op.match(LI->getPointerOperand()); + return false; + } +}; + +/// Matches LoadInst. +template <typename OpTy> inline LoadClass_match<OpTy> m_Load(const OpTy &Op) { + return LoadClass_match<OpTy>(Op); +} +//===----------------------------------------------------------------------===// // Matchers for unary operators // diff --git a/contrib/llvm/include/llvm/IR/Type.h b/contrib/llvm/include/llvm/IR/Type.h index ef7801266777..1574fc334ffc 100644 --- a/contrib/llvm/include/llvm/IR/Type.h +++ b/contrib/llvm/include/llvm/IR/Type.h @@ -438,7 +438,7 @@ private: }; // Printing of types. -static inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) { +inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) { T.print(OS); return OS; } diff --git a/contrib/llvm/include/llvm/IR/Value.def b/contrib/llvm/include/llvm/IR/Value.def index cebd7f7297ef..e2ddba0aa159 100644 --- a/contrib/llvm/include/llvm/IR/Value.def +++ b/contrib/llvm/include/llvm/IR/Value.def @@ -56,16 +56,10 @@ #define HANDLE_CONSTANT_MARKER(MarkerName, ValueName) #endif -HANDLE_VALUE(Argument) -HANDLE_VALUE(BasicBlock) - -// FIXME: It's awkward that Value.def knows about classes in Analysis. While -// this doesn't introduce a strict link or include dependency, we should remove -// the circular dependency eventually. -HANDLE_MEMORY_VALUE(MemoryUse) -HANDLE_MEMORY_VALUE(MemoryDef) -HANDLE_MEMORY_VALUE(MemoryPhi) +// Having constant first makes the range check for isa<Constant> faster +// and smaller by one operation. +// Constant HANDLE_GLOBAL_VALUE(Function) HANDLE_GLOBAL_VALUE(GlobalAlias) HANDLE_GLOBAL_VALUE(GlobalIFunc) @@ -88,13 +82,6 @@ HANDLE_CONSTANT(ConstantFP) HANDLE_CONSTANT(ConstantPointerNull) HANDLE_CONSTANT(ConstantTokenNone) -HANDLE_METADATA_VALUE(MetadataAsValue) -HANDLE_INLINE_ASM_VALUE(InlineAsm) - -HANDLE_INSTRUCTION(Instruction) -// Enum values starting at InstructionVal are used for Instructions; -// don't add new values here! - HANDLE_CONSTANT_MARKER(ConstantFirstVal, Function) HANDLE_CONSTANT_MARKER(ConstantLastVal, ConstantTokenNone) HANDLE_CONSTANT_MARKER(ConstantDataFirstVal, UndefValue) @@ -102,6 +89,24 @@ HANDLE_CONSTANT_MARKER(ConstantDataLastVal, ConstantTokenNone) HANDLE_CONSTANT_MARKER(ConstantAggregateFirstVal, ConstantArray) HANDLE_CONSTANT_MARKER(ConstantAggregateLastVal, ConstantVector) +HANDLE_VALUE(Argument) +HANDLE_VALUE(BasicBlock) + + +HANDLE_METADATA_VALUE(MetadataAsValue) +HANDLE_INLINE_ASM_VALUE(InlineAsm) + +// FIXME: It's awkward that Value.def knows about classes in Analysis. While +// this doesn't introduce a strict link or include dependency, we should remove +// the circular dependency eventually. +HANDLE_MEMORY_VALUE(MemoryUse) +HANDLE_MEMORY_VALUE(MemoryDef) +HANDLE_MEMORY_VALUE(MemoryPhi) + +HANDLE_INSTRUCTION(Instruction) +// Enum values starting at InstructionVal are used for Instructions; +// don't add new values here! + #undef HANDLE_MEMORY_VALUE #undef HANDLE_GLOBAL_VALUE #undef HANDLE_CONSTANT diff --git a/contrib/llvm/include/llvm/IR/Value.h b/contrib/llvm/include/llvm/IR/Value.h index 9e4914973edf..d848fe921868 100644 --- a/contrib/llvm/include/llvm/IR/Value.h +++ b/contrib/llvm/include/llvm/IR/Value.h @@ -299,6 +299,12 @@ public: /// values or constant users. void replaceUsesOutsideBlock(Value *V, BasicBlock *BB); + /// replaceUsesExceptBlockAddr - Go through the uses list for this definition + /// and make each use point to "V" instead of "this" when the use is outside + /// the block. 'This's use list is expected to have at least one element. + /// Unlike replaceAllUsesWith this function skips blockaddr uses. + void replaceUsesExceptBlockAddr(Value *New); + //---------------------------------------------------------------------- // Methods for handling the chain of uses of this Value. // @@ -324,6 +330,10 @@ public: return UseList == nullptr; } + bool materialized_use_empty() const { + return UseList == nullptr; + } + using use_iterator = use_iterator_impl<Use>; using const_use_iterator = use_iterator_impl<const Use>; @@ -560,7 +570,7 @@ public: /// /// If CanBeNull is set by this function the pointer can either be null or be /// dereferenceable up to the returned number of bytes. - unsigned getPointerDereferenceableBytes(const DataLayout &DL, + uint64_t getPointerDereferenceableBytes(const DataLayout &DL, bool &CanBeNull) const; /// \brief Returns an alignment of the pointer value. @@ -645,12 +655,6 @@ private: return Merged; } - /// \brief Tail-recursive helper for \a mergeUseLists(). - /// - /// \param[out] Next the first element in the list. - template <class Compare> - static void mergeUseListsImpl(Use *L, Use *R, Use **Next, Compare Cmp); - protected: unsigned short getSubclassDataFromValue() const { return SubclassData; } void setValueSubclassData(unsigned short D) { SubclassData = D; } @@ -756,8 +760,8 @@ template <class Compare> void Value::sortUseList(Compare Cmp) { // template <> struct isa_impl<Constant, Value> { static inline bool doit(const Value &Val) { - return Val.getValueID() >= Value::ConstantFirstVal && - Val.getValueID() <= Value::ConstantLastVal; + static_assert(Value::ConstantFirstVal == 0, "Val.getValueID() >= Value::ConstantFirstVal"); + return Val.getValueID() <= Value::ConstantLastVal; } }; diff --git a/contrib/llvm/include/llvm/IR/Verifier.h b/contrib/llvm/include/llvm/IR/Verifier.h index 15e52d9e0742..bc10f330bc8a 100644 --- a/contrib/llvm/include/llvm/IR/Verifier.h +++ b/contrib/llvm/include/llvm/IR/Verifier.h @@ -61,11 +61,13 @@ class TBAAVerifier { /// \name Helper functions used by \c visitTBAAMetadata. /// @{ MDNode *getFieldNodeFromTBAABaseNode(Instruction &I, const MDNode *BaseNode, - APInt &Offset); + APInt &Offset, bool IsNewFormat); TBAAVerifier::TBAABaseNodeSummary verifyTBAABaseNode(Instruction &I, - const MDNode *BaseNode); + const MDNode *BaseNode, + bool IsNewFormat); TBAABaseNodeSummary verifyTBAABaseNodeImpl(Instruction &I, - const MDNode *BaseNode); + const MDNode *BaseNode, + bool IsNewFormat); bool isValidScalarTBAANode(const MDNode *MD); /// @} diff --git a/contrib/llvm/include/llvm/IRReader/IRReader.h b/contrib/llvm/include/llvm/IRReader/IRReader.h index 7b24ec11fb64..f5621647db06 100644 --- a/contrib/llvm/include/llvm/IRReader/IRReader.h +++ b/contrib/llvm/include/llvm/IRReader/IRReader.h @@ -37,14 +37,22 @@ getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, /// If the given MemoryBuffer holds a bitcode image, return a Module /// for it. Otherwise, attempt to parse it as LLVM Assembly and return /// a Module for it. +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! std::unique_ptr<Module> parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, - LLVMContext &Context); + LLVMContext &Context, + bool UpgradeDebugInfo = true); /// If the given file holds a bitcode image, return a Module for it. /// Otherwise, attempt to parse it as LLVM Assembly and return a Module /// for it. +/// \param UpgradeDebugInfo Run UpgradeDebugInfo, which runs the Verifier. +/// This option should only be set to false by llvm-as +/// for use inside the LLVM testuite! std::unique_ptr<Module> parseIRFile(StringRef Filename, SMDiagnostic &Err, - LLVMContext &Context); + LLVMContext &Context, + bool UpgradeDebugInfo = true); } #endif diff --git a/contrib/llvm/include/llvm/InitializePasses.h b/contrib/llvm/include/llvm/InitializePasses.h index 39ac4649b70d..dd7aa722ed2b 100644 --- a/contrib/llvm/include/llvm/InitializePasses.h +++ b/contrib/llvm/include/llvm/InitializePasses.h @@ -1,4 +1,4 @@ -//===- llvm/InitializePasses.h -------- Initialize All Passes ---*- C++ -*-===// +//===- llvm/InitializePasses.h - Initialize All Passes ----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -75,12 +75,12 @@ void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); void initializeBlockExtractorPassPass(PassRegistry&); void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&); -void initializeBoundsCheckingPass(PassRegistry&); -void initializeBranchCoalescingPass(PassRegistry&); +void initializeBoundsCheckingLegacyPassPass(PassRegistry&); void initializeBranchFolderPassPass(PassRegistry&); void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&); void initializeBranchRelaxationPass(PassRegistry&); void initializeBreakCriticalEdgesPass(PassRegistry&); +void initializeCallSiteSplittingLegacyPassPass(PassRegistry&); void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&); void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&); void initializeCFGPrinterLegacyPassPass(PassRegistry&); @@ -94,11 +94,13 @@ void initializeCallGraphViewerPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry&); void initializeCodeGenPreparePass(PassRegistry&); void initializeConstantHoistingLegacyPassPass(PassRegistry&); +void initializeCalledValuePropagationLegacyPassPass(PassRegistry &); void initializeConstantMergeLegacyPassPass(PassRegistry&); void initializeConstantPropagationPass(PassRegistry&); void initializeCorrelatedValuePropagationPass(PassRegistry&); void initializeCostModelAnalysisPass(PassRegistry&); -void initializeCountingFunctionInserterPass(PassRegistry&); +void initializeEntryExitInstrumenterPass(PassRegistry&); +void initializePostInlineEntryExitInstrumenterPass(PassRegistry&); void initializeCrossDSOCFIPass(PassRegistry&); void initializeDAEPass(PassRegistry&); void initializeDAHPass(PassRegistry&); @@ -113,6 +115,7 @@ void initializeDependenceAnalysisPass(PassRegistry&); void initializeDependenceAnalysisWrapperPassPass(PassRegistry&); void initializeDetectDeadLanesPass(PassRegistry&); void initializeDivergenceAnalysisPass(PassRegistry&); +void initializeDivRemPairsLegacyPassPass(PassRegistry&); void initializeDomOnlyPrinterPass(PassRegistry&); void initializeDomOnlyViewerPass(PassRegistry&); void initializeDomPrinterPass(PassRegistry&); @@ -127,6 +130,7 @@ void initializeEdgeBundlesPass(PassRegistry&); void initializeEfficiencySanitizerPass(PassRegistry&); void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry&); void initializeExpandISelPseudosPass(PassRegistry&); +void initializeExpandMemCmpPassPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); void initializeExpandReductionsPass(PassRegistry&); void initializeExternalAAWrapperPassPass(PassRegistry&); @@ -173,7 +177,6 @@ void initializeIntervalPartitionPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); -void initializeLateCFGSimplifyPassPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&); void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&); @@ -256,6 +259,7 @@ void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerPass(PassRegistry&); void initializeMergeFunctionsPass(PassRegistry&); +void initializeMergeICmpsPass(PassRegistry&); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); @@ -303,7 +307,7 @@ void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&); void initializePromoteLegacyPassPass(PassRegistry&); void initializePruneEHPass(PassRegistry&); void initializeRABasicPass(PassRegistry&); -void initializeRAFastPass(PassRegistry&); +void initializeRegAllocFastPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); @@ -317,7 +321,7 @@ void initializeRegisterCoalescerPass(PassRegistry&); void initializeRenameIndependentSubregsPass(PassRegistry&); void initializeResetMachineFunctionPass(PassRegistry&); void initializeReversePostOrderFunctionAttrsLegacyPassPass(PassRegistry&); -void initializeRewriteStatepointsForGCPass(PassRegistry&); +void initializeRewriteStatepointsForGCLegacyPassPass(PassRegistry &); void initializeRewriteSymbolsLegacyPassPass(PassRegistry&); void initializeSafepointIRVerifierPass(PassRegistry&); void initializeSCCPLegacyPassPass(PassRegistry&); @@ -356,6 +360,7 @@ void initializeStripNonDebugSymbolsPass(PassRegistry&); void initializeStripNonLineTableDebugInfoPass(PassRegistry&); void initializeStripSymbolsPass(PassRegistry&); void initializeStructurizeCFGPass(PassRegistry&); +void initializeHWAddressSanitizerPass(PassRegistry&); void initializeTailCallElimPass(PassRegistry&); void initializeTailDuplicatePassPass(PassRegistry&); void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&); @@ -376,6 +381,8 @@ void initializeWinEHPreparePass(PassRegistry&); void initializeWriteBitcodePassPass(PassRegistry&); void initializeWriteThinLTOBitcodePass(PassRegistry&); void initializeXRayInstrumentationPass(PassRegistry&); -} +void initializeMIRCanonicalizerPass(PassRegistry &); -#endif +} // end namespace llvm + +#endif // LLVM_INITIALIZEPASSES_H diff --git a/contrib/llvm/include/llvm/LTO/Caching.h b/contrib/llvm/include/llvm/LTO/Caching.h index f5ec70e081c1..25c719a68b92 100644 --- a/contrib/llvm/include/llvm/LTO/Caching.h +++ b/contrib/llvm/include/llvm/LTO/Caching.h @@ -24,12 +24,13 @@ namespace lto { /// This type defines the callback to add a pre-existing native object file /// (e.g. in a cache). /// -/// MB->getBufferIdentifier() is a valid path for the file at the time that it -/// was opened, but clients should prefer to access MB directly in order to -/// avoid a potential race condition. +/// Path is generally expected to be a valid path for the file at the point when +/// the AddBufferFn function is called, but clients should prefer to access MB +/// directly in order to avoid a potential race condition. /// /// Buffer callbacks must be thread safe. -typedef std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)> +typedef std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB, + StringRef Path)> AddBufferFn; /// Create a local file system cache which uses the given cache directory and diff --git a/contrib/llvm/include/llvm/LTO/Config.h b/contrib/llvm/include/llvm/LTO/Config.h index 73106f77ca55..4bd981c090b1 100644 --- a/contrib/llvm/include/llvm/LTO/Config.h +++ b/contrib/llvm/include/llvm/LTO/Config.h @@ -40,7 +40,7 @@ struct Config { TargetOptions Options; std::vector<std::string> MAttrs; Optional<Reloc::Model> RelocModel = Reloc::PIC_; - CodeModel::Model CodeModel = CodeModel::Default; + Optional<CodeModel::Model> CodeModel = None; CodeGenOpt::Level CGOptLevel = CodeGenOpt::Default; TargetMachine::CodeGenFileType CGFileType = TargetMachine::CGFT_ObjectFile; unsigned OptLevel = 2; @@ -79,6 +79,9 @@ struct Config { /// Whether to emit optimization remarks with hotness informations. bool RemarksWithHotness = false; + /// Whether to emit the pass manager debuggging informations. + bool DebugPassManager = false; + bool ShouldDiscardValueNames = true; DiagnosticHandlerFunction DiagHandler; @@ -168,20 +171,27 @@ struct Config { bool UseInputModulePath = false); }; +struct LTOLLVMDiagnosticHandler : public DiagnosticHandler { + DiagnosticHandlerFunction *Fn; + LTOLLVMDiagnosticHandler(DiagnosticHandlerFunction *DiagHandlerFn) + : Fn(DiagHandlerFn) {} + bool handleDiagnostics(const DiagnosticInfo &DI) override { + (*Fn)(DI); + return true; + } +}; /// A derived class of LLVMContext that initializes itself according to a given /// Config object. The purpose of this class is to tie ownership of the /// diagnostic handler to the context, as opposed to the Config object (which /// may be ephemeral). +// FIXME: This should not be required as diagnostic handler is not callback. struct LTOLLVMContext : LLVMContext { - static void funcDiagHandler(const DiagnosticInfo &DI, void *Context) { - auto *Fn = static_cast<DiagnosticHandlerFunction *>(Context); - (*Fn)(DI); - } LTOLLVMContext(const Config &C) : DiagHandler(C.DiagHandler) { setDiscardValueNames(C.ShouldDiscardValueNames); enableDebugTypeODRUniquing(); - setDiagnosticHandler(funcDiagHandler, &DiagHandler, true); + setDiagnosticHandler( + llvm::make_unique<LTOLLVMDiagnosticHandler>(&DiagHandler), true); } DiagnosticHandlerFunction DiagHandler; }; diff --git a/contrib/llvm/include/llvm/LTO/LTO.h b/contrib/llvm/include/llvm/LTO/LTO.h index d678a68ed860..2a2b59847281 100644 --- a/contrib/llvm/include/llvm/LTO/LTO.h +++ b/contrib/llvm/include/llvm/LTO/LTO.h @@ -71,7 +71,7 @@ std::string getThinLTOOutputFile(const std::string &Path, const std::string &NewPrefix); /// Setup optimization remarks. -Expected<std::unique_ptr<tool_output_file>> +Expected<std::unique_ptr<ToolOutputFile>> setupOptimizationRemarks(LLVMContext &Context, StringRef LTORemarksFilename, bool LTOPassRemarksWithHotness, int Count = -1); @@ -126,6 +126,7 @@ public: using irsymtab::Symbol::getCommonSize; using irsymtab::Symbol::getCommonAlignment; using irsymtab::Symbol::getCOFFWeakExternalFallback; + using irsymtab::Symbol::getSectionName; using irsymtab::Symbol::isExecutable; }; @@ -278,7 +279,6 @@ private: unsigned ParallelCodeGenParallelismLevel; LTOLLVMContext Ctx; - bool HasModule = false; std::unique_ptr<Module> CombinedModule; std::unique_ptr<IRMover> Mover; @@ -371,8 +371,7 @@ private: const SymbolResolution *&ResI, const SymbolResolution *ResE); Error runRegularLTO(AddStreamFn AddStream); - Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, - bool HasRegularLTO); + Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache); mutable bool CalledGetMaxTasks = false; }; diff --git a/contrib/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/contrib/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h index 952875fc854e..f48ab02863a5 100644 --- a/contrib/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/contrib/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -184,6 +184,7 @@ struct LTOCodeGenerator { LLVMContext &getContext() { return Context; } void resetMergedModule() { MergedModule.reset(); } + void DiagnosticHandler(const DiagnosticInfo &DI); private: void initializeLTOPasses(); @@ -204,10 +205,6 @@ private: bool determineTarget(); std::unique_ptr<TargetMachine> createTargetMachine(); - static void DiagnosticHandler(const DiagnosticInfo &DI, void *Context); - - void DiagnosticHandler2(const DiagnosticInfo &DI); - void emitError(const std::string &ErrMsg); void emitWarning(const std::string &ErrMsg); @@ -240,7 +237,7 @@ private: bool ShouldEmbedUselists = false; bool ShouldRestoreGlobalsLinkage = false; TargetMachine::CodeGenFileType FileType = TargetMachine::CGFT_ObjectFile; - std::unique_ptr<tool_output_file> DiagnosticOutputFile; + std::unique_ptr<ToolOutputFile> DiagnosticOutputFile; bool Freestanding = false; }; } diff --git a/contrib/llvm/include/llvm/LinkAllIR.h b/contrib/llvm/include/llvm/LinkAllIR.h index de1d305f8e77..9a9f3d3a677f 100644 --- a/contrib/llvm/include/llvm/LinkAllIR.h +++ b/contrib/llvm/include/llvm/LinkAllIR.h @@ -19,7 +19,6 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" diff --git a/contrib/llvm/include/llvm/LinkAllPasses.h b/contrib/llvm/include/llvm/LinkAllPasses.h index d07c15c1013b..39d1ec6cffb5 100644 --- a/contrib/llvm/include/llvm/LinkAllPasses.h +++ b/contrib/llvm/include/llvm/LinkAllPasses.h @@ -43,6 +43,7 @@ #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/BoundsChecking.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/GVN.h" @@ -70,16 +71,16 @@ namespace { (void) llvm::createSCEVAAWrapperPass(); (void) llvm::createTypeBasedAAWrapperPass(); (void) llvm::createScopedNoAliasAAWrapperPass(); - (void) llvm::createBoundsCheckingPass(); + (void) llvm::createBoundsCheckingLegacyPass(); (void) llvm::createBreakCriticalEdgesPass(); (void) llvm::createCallGraphDOTPrinterPass(); (void) llvm::createCallGraphViewerPass(); (void) llvm::createCFGSimplificationPass(); - (void) llvm::createLateCFGSimplificationPass(); (void) llvm::createCFLAndersAAWrapperPass(); (void) llvm::createCFLSteensAAWrapperPass(); (void) llvm::createStructurizeCFGPass(); (void) llvm::createLibCallsShrinkWrapPass(); + (void) llvm::createCalledValuePropagationPass(); (void) llvm::createConstantMergePass(); (void) llvm::createConstantPropagationPass(); (void) llvm::createCostModelAnalysisPass(); @@ -165,7 +166,8 @@ namespace { (void) llvm::createInstCountPass(); (void) llvm::createConstantHoistingPass(); (void) llvm::createCodeGenPreparePass(); - (void) llvm::createCountingFunctionInserterPass(); + (void) llvm::createEntryExitInstrumenterPass(); + (void) llvm::createPostInlineEntryExitInstrumenterPass(); (void) llvm::createEarlyCSEPass(); (void) llvm::createGVNHoistPass(); (void) llvm::createMergedLoadStoreMotionPass(); @@ -179,6 +181,8 @@ namespace { (void) llvm::createPostOrderFunctionAttrsLegacyPass(); (void) llvm::createReversePostOrderFunctionAttrsPass(); (void) llvm::createMergeFunctionsPass(); + (void) llvm::createMergeICmpsPass(); + (void) llvm::createExpandMemCmpPass(); std::string buf; llvm::raw_string_ostream os(buf); (void) llvm::createPrintModulePass(os); diff --git a/contrib/llvm/include/llvm/MC/LaneBitmask.h b/contrib/llvm/include/llvm/MC/LaneBitmask.h index 73b987b074db..8c0b4ecb8fd4 100644 --- a/contrib/llvm/include/llvm/MC/LaneBitmask.h +++ b/contrib/llvm/include/llvm/MC/LaneBitmask.h @@ -73,9 +73,16 @@ namespace llvm { constexpr Type getAsInteger() const { return Mask; } - static LaneBitmask getNone() { return LaneBitmask(0); } - static LaneBitmask getAll() { return ~LaneBitmask(0); } - static LaneBitmask getLane(unsigned Lane) { + unsigned getNumLanes() const { + return countPopulation(Mask); + } + unsigned getHighestLane() const { + return Log2_32(Mask); + } + + static constexpr LaneBitmask getNone() { return LaneBitmask(0); } + static constexpr LaneBitmask getAll() { return ~LaneBitmask(0); } + static constexpr LaneBitmask getLane(unsigned Lane) { return LaneBitmask(Type(1) << Lane); } @@ -84,7 +91,7 @@ namespace llvm { }; /// Create Printable object to print LaneBitmasks on a \ref raw_ostream. - static LLVM_ATTRIBUTE_UNUSED Printable PrintLaneMask(LaneBitmask LaneMask) { + inline Printable PrintLaneMask(LaneBitmask LaneMask) { return Printable([LaneMask](raw_ostream &OS) { OS << format(LaneBitmask::FormatStr, LaneMask.getAsInteger()); }); diff --git a/contrib/llvm/include/llvm/MC/MCAsmBackend.h b/contrib/llvm/include/llvm/MC/MCAsmBackend.h index 5a8e29d08ad2..ef2007ff6920 100644 --- a/contrib/llvm/include/llvm/MC/MCAsmBackend.h +++ b/contrib/llvm/include/llvm/MC/MCAsmBackend.h @@ -15,17 +15,22 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCFragment.h" #include <cstdint> +#include <memory> namespace llvm { class MCAsmLayout; class MCAssembler; class MCCFIInstruction; +class MCCodePadder; struct MCFixupKindInfo; class MCFragment; class MCInst; +class MCObjectStreamer; class MCObjectWriter; +struct MCCodePaddingContext; class MCRelaxableFragment; class MCSubtargetInfo; class MCValue; @@ -33,8 +38,11 @@ class raw_pwrite_stream; /// Generic interface to target specific assembler backends. class MCAsmBackend { + std::unique_ptr<MCCodePadder> CodePadder; + protected: // Can only create subclasses. MCAsmBackend(); + MCAsmBackend(std::unique_ptr<MCCodePadder> TargetCodePadder); public: MCAsmBackend(const MCAsmBackend &) = delete; @@ -46,7 +54,8 @@ public: /// Create a new MCObjectWriter instance for use by the assembler backend to /// emit the final object file. - virtual MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const = 0; + virtual std::unique_ptr<MCObjectWriter> + createObjectWriter(raw_pwrite_stream &OS) const = 0; /// \name Target Fixup Interfaces /// @{ @@ -132,6 +141,40 @@ public: generateCompactUnwindEncoding(ArrayRef<MCCFIInstruction>) const { return 0; } + + /// Handles all target related code padding when starting to write a new + /// basic block to an object file. + /// + /// \param OS The streamer used for writing the padding data and function. + /// \param Context the context of the padding, Embeds the basic block's + /// parameters. + void handleCodePaddingBasicBlockStart(MCObjectStreamer *OS, + const MCCodePaddingContext &Context); + /// Handles all target related code padding after writing a block to an object + /// file. + /// + /// \param Context the context of the padding, Embeds the basic block's + /// parameters. + void handleCodePaddingBasicBlockEnd(const MCCodePaddingContext &Context); + /// Handles all target related code padding before writing a new instruction + /// to an object file. + /// + /// \param Inst the instruction. + void handleCodePaddingInstructionBegin(const MCInst &Inst); + /// Handles all target related code padding after writing an instruction to an + /// object file. + /// + /// \param Inst the instruction. + void handleCodePaddingInstructionEnd(const MCInst &Inst); + + /// Relaxes a fragment (changes the size of the padding) according to target + /// requirements. The new size computation is done w.r.t a layout. + /// + /// \param PF The fragment to relax. + /// \param Layout Code layout information. + /// + /// \returns true iff any relaxation occured. + bool relaxFragment(MCPaddingFragment *PF, MCAsmLayout &Layout); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCAssembler.h b/contrib/llvm/include/llvm/MC/MCAssembler.h index 4f1b5a8b3d72..034605557d4c 100644 --- a/contrib/llvm/include/llvm/MC/MCAssembler.h +++ b/contrib/llvm/include/llvm/MC/MCAssembler.h @@ -16,6 +16,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/BinaryFormat/MachO.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCFixup.h" @@ -84,8 +85,12 @@ public: /// MachO specific deployment target version info. // A Major version of 0 indicates that no version information was supplied // and so the corresponding load command should not be emitted. - using VersionMinInfoType = struct { - MCVersionMinType Kind; + using VersionInfoType = struct { + bool EmitBuildVersion; + union { + MCVersionMinType Type; ///< Used when EmitBuildVersion==false. + MachO::PlatformType Platform; ///< Used when EmitBuildVersion==true. + } TypeOrPlatform; unsigned Major; unsigned Minor; unsigned Update; @@ -145,7 +150,7 @@ private: /// the Streamer and the .o writer MCLOHContainer LOHContainer; - VersionMinInfoType VersionMinInfo; + VersionInfoType VersionInfo; /// Evaluate a fixup to a relocatable expression and the value which should be /// placed into the fixup. @@ -183,6 +188,8 @@ private: bool relaxInstruction(MCAsmLayout &Layout, MCRelaxableFragment &IF); + bool relaxPaddingFragment(MCAsmLayout &Layout, MCPaddingFragment &PF); + bool relaxLEB(MCAsmLayout &Layout, MCLEBFragment &IF); bool relaxDwarfLineAddr(MCAsmLayout &Layout, MCDwarfLineAddrFragment &DF); @@ -241,13 +248,22 @@ public: void setELFHeaderEFlags(unsigned Flags) { ELFHeaderEFlags = Flags; } /// MachO deployment target version information. - const VersionMinInfoType &getVersionMinInfo() const { return VersionMinInfo; } - void setVersionMinInfo(MCVersionMinType Kind, unsigned Major, unsigned Minor, - unsigned Update) { - VersionMinInfo.Kind = Kind; - VersionMinInfo.Major = Major; - VersionMinInfo.Minor = Minor; - VersionMinInfo.Update = Update; + const VersionInfoType &getVersionInfo() const { return VersionInfo; } + void setVersionMin(MCVersionMinType Type, unsigned Major, unsigned Minor, + unsigned Update) { + VersionInfo.EmitBuildVersion = false; + VersionInfo.TypeOrPlatform.Type = Type; + VersionInfo.Major = Major; + VersionInfo.Minor = Minor; + VersionInfo.Update = Update; + } + void setBuildVersion(MachO::PlatformType Platform, unsigned Major, + unsigned Minor, unsigned Update) { + VersionInfo.EmitBuildVersion = true; + VersionInfo.TypeOrPlatform.Platform = Platform; + VersionInfo.Major = Major; + VersionInfo.Minor = Minor; + VersionInfo.Update = Update; } /// Reuse an assembler instance diff --git a/contrib/llvm/include/llvm/MC/MCCodePadder.h b/contrib/llvm/include/llvm/MC/MCCodePadder.h new file mode 100644 index 000000000000..1e91198597c3 --- /dev/null +++ b/contrib/llvm/include/llvm/MC/MCCodePadder.h @@ -0,0 +1,243 @@ +//===- llvm/MC/CodePadder.h - MC Code Padder --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCCODEPADDER_H +#define LLVM_MC_MCCODEPADDER_H + +#include "MCFragment.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" + +namespace llvm { + +class MCAsmLayout; +class MCCodePaddingPolicy; +class MCFragment; +class MCInst; +class MCObjectStreamer; +class MCSection; + +typedef SmallVector<const MCPaddingFragment *, 8> MCPFRange; + +struct MCCodePaddingContext { + bool IsPaddingActive; + bool IsBasicBlockInsideInnermostLoop; + bool IsBasicBlockReachableViaFallthrough; + bool IsBasicBlockReachableViaBranch; +}; + +/// Target-independent base class incharge of all code padding decisions for a +/// target. During encoding it determines if and where MCPaddingFragments will +/// be located, as later on, when layout information is available, it determines +/// their sizes. +class MCCodePadder { + MCCodePadder(const MCCodePadder &) = delete; + void operator=(const MCCodePadder &) = delete; + + /// Determines if the MCCodePaddingPolicies are active. + bool ArePoliciesActive; + + /// All the supported MCCodePaddingPolicies. + SmallPtrSet<MCCodePaddingPolicy *, 4> CodePaddingPolicies; + + /// A pointer to the fragment of the instruction whose padding is currently + /// done for. + MCPaddingFragment *CurrHandledInstFragment; + + /// A map holding the jurisdiction for each padding fragment. Key: padding + /// fragment. Value: The fragment's jurisdiction. A jurisdiction is a vector + /// of padding fragments whose conditions are being controlled by another + /// fragment, the key fragment. + DenseMap<MCPaddingFragment *, MCPFRange> FragmentToJurisdiction; + MCPFRange &getJurisdiction(MCPaddingFragment *Fragment, MCAsmLayout &Layout); + + /// A map holding the maximal instruction window size relevant for a padding + /// fragment. + DenseMap<MCPaddingFragment *, uint64_t> FragmentToMaxWindowSize; + uint64_t getMaxWindowSize(MCPaddingFragment *Fragment, MCAsmLayout &Layout); + +protected: + /// The current streamer, used to stream code padding. + MCObjectStreamer *OS; + + bool addPolicy(MCCodePaddingPolicy *Policy); + + virtual bool + basicBlockRequiresInsertionPoint(const MCCodePaddingContext &Context) { + return false; + } + + virtual bool instructionRequiresInsertionPoint(const MCInst &Inst) { + return false; + } + + virtual bool usePoliciesForBasicBlock(const MCCodePaddingContext &Context) { + return Context.IsPaddingActive; + } + +public: + MCCodePadder() + : ArePoliciesActive(false), CurrHandledInstFragment(nullptr), + OS(nullptr) {} + virtual ~MCCodePadder(); + + /// Handles all target related code padding when starting to write a new + /// basic block to an object file. + /// + /// \param OS The streamer used for writing the padding data and function. + /// \param Context the context of the padding, Embeds the basic block's + /// parameters. + void handleBasicBlockStart(MCObjectStreamer *OS, + const MCCodePaddingContext &Context); + /// Handles all target related code padding when done writing a block to an + /// object file. + /// + /// \param Context the context of the padding, Embeds the basic block's + /// parameters. + void handleBasicBlockEnd(const MCCodePaddingContext &Context); + /// Handles all target related code padding before writing a new instruction + /// to an object file. + /// + /// \param Inst the instruction. + void handleInstructionBegin(const MCInst &Inst); + /// Handles all target related code padding after writing an instruction to an + /// object file. + /// + /// \param Inst the instruction. + void handleInstructionEnd(const MCInst &Inst); + + /// Relaxes a fragment (changes the size of the padding) according to target + /// requirements. The new size computation is done w.r.t a layout. + /// + /// \param Fragment The fragment to relax. + /// \param Layout Code layout information. + /// + /// \returns true iff any relaxation occured. + bool relaxFragment(MCPaddingFragment *Fragment, MCAsmLayout &Layout); +}; + +/// The base class for all padding policies, i.e. a rule or set of rules to pad +/// the generated code. +class MCCodePaddingPolicy { + MCCodePaddingPolicy() = delete; + MCCodePaddingPolicy(const MCCodePaddingPolicy &) = delete; + void operator=(const MCCodePaddingPolicy &) = delete; + +protected: + /// A mask holding the kind of this policy, i.e. only the i'th bit will be set + /// where i is the kind number. + const uint64_t KindMask; + /// Instruction window size relevant to this policy. + const uint64_t WindowSize; + /// A boolean indicating which byte of the instruction determies its + /// instruction window. If true - the last byte of the instructions, o.w. - + /// the first byte of the instruction. + const bool InstByteIsLastByte; + + MCCodePaddingPolicy(uint64_t Kind, uint64_t WindowSize, + bool InstByteIsLastByte) + : KindMask(UINT64_C(1) << Kind), WindowSize(WindowSize), + InstByteIsLastByte(InstByteIsLastByte) {} + + /// Computes and returns the offset of the consecutive fragment of a given + /// fragment. + /// + /// \param Fragment The fragment whose consecutive offset will be computed. + /// \param Layout Code layout information. + /// + /// \returns the offset of the consecutive fragment of \p Fragment. + static uint64_t getNextFragmentOffset(const MCFragment *Fragment, + const MCAsmLayout &Layout); + /// Returns the instruction byte of an instruction pointed by a given + /// MCPaddingFragment. An instruction byte is the address of the byte of an + /// instruction which determines its instruction window. + /// + /// \param Fragment The fragment pointing to the instruction. + /// \param Layout Code layout information. + /// + /// \returns the instruction byte of an instruction pointed by \p Fragment. + uint64_t getFragmentInstByte(const MCPaddingFragment *Fragment, + MCAsmLayout &Layout) const; + uint64_t computeWindowEndAddress(const MCPaddingFragment *Fragment, + uint64_t Offset, MCAsmLayout &Layout) const; + + /// Computes and returns the penalty weight of a first instruction window in a + /// range. This requires a special function since the first window does not + /// contain all the padding fragments in that window. It only contains all the + /// padding fragments starting from the relevant insertion point. + /// + /// \param Window The first window. + /// \param Offset The offset of the parent section relative to the beginning + /// of the file, mod the window size. + /// \param Layout Code layout information. + /// + /// \returns the penalty weight of a first instruction window in a range, \p + /// Window. + double computeFirstWindowPenaltyWeight(const MCPFRange &Window, + uint64_t Offset, + MCAsmLayout &Layout) const; + /// Computes and returns the penalty caused by an instruction window. + /// + /// \param Window The instruction window. + /// \param Offset The offset of the parent section relative to the beginning + /// of the file, mod the window size. + /// \param Layout Code layout information. + /// + /// \returns the penalty caused by \p Window. + virtual double computeWindowPenaltyWeight(const MCPFRange &Window, + uint64_t Offset, + MCAsmLayout &Layout) const = 0; + +public: + virtual ~MCCodePaddingPolicy() {} + + /// Returns the kind mask of this policy - A mask holding the kind of this + /// policy, i.e. only the i'th bit will be set where i is the kind number. + uint64_t getKindMask() const { return KindMask; } + /// Returns the instruction window size relevant to this policy. + uint64_t getWindowSize() const { return WindowSize; } + /// Returns true if the last byte of an instruction determines its instruction + /// window, or false if the first of an instruction determines it. + bool isInstByteLastByte() const { return InstByteIsLastByte; } + + /// Returns true iff this policy needs padding for a given basic block. + /// + /// \param Context the context of the padding, Embeds the basic block's + /// parameters. + /// + /// \returns true iff this policy needs padding for the basic block. + virtual bool + basicBlockRequiresPaddingFragment(const MCCodePaddingContext &Context) const { + return false; + } + /// Returns true iff this policy needs padding for a given instruction. + /// + /// \param Inst The given instruction. + /// + /// \returns true iff this policy needs padding for \p Inst. + virtual bool instructionRequiresPaddingFragment(const MCInst &Inst) const { + return false; + } + /// Computes and returns the penalty caused by a range of instruction windows. + /// The weight is computed for each window separelty and then accumulated. + /// + /// \param Range The range. + /// \param Offset The offset of the parent section relative to the beginning + /// of the file, mod the window size. + /// \param Layout Code layout information. + /// + /// \returns the penalty caused by \p Range. + double computeRangePenaltyWeight(const MCPFRange &Range, uint64_t Offset, + MCAsmLayout &Layout) const; +}; + +} // namespace llvm + +#endif // LLVM_MC_MCCODEPADDER_H diff --git a/contrib/llvm/include/llvm/MC/MCCodeView.h b/contrib/llvm/include/llvm/MC/MCCodeView.h index c3f1cecc97f4..e2249f49c86c 100644 --- a/contrib/llvm/include/llvm/MC/MCCodeView.h +++ b/contrib/llvm/include/llvm/MC/MCCodeView.h @@ -161,8 +161,8 @@ public: ~CodeViewContext(); bool isValidFileNumber(unsigned FileNumber) const; - bool addFile(unsigned FileNumber, StringRef Filename); - ArrayRef<StringRef> getFilenames() { return Filenames; } + bool addFile(MCStreamer &OS, unsigned FileNumber, StringRef Filename, + ArrayRef<uint8_t> ChecksumBytes, uint8_t ChecksumKind); /// Records the function id of a normal function. Returns false if the /// function id has already been used, and true otherwise. @@ -273,6 +273,13 @@ public: /// Emits the file checksum substream. void emitFileChecksums(MCObjectStreamer &OS); + /// Emits the offset into the checksum table of the given file number. + void emitFileChecksumOffset(MCObjectStreamer &OS, unsigned FileNo); + + /// Add something to the string table. Returns the final string as well as + /// offset into the string table. + std::pair<StringRef, unsigned> addToStringTable(StringRef S); + private: /// The current CodeView line information from the last .cv_loc directive. MCCVLoc CurrentCVLoc = MCCVLoc(0, 0, 0, 0, false, true); @@ -287,14 +294,27 @@ private: MCDataFragment *getStringTableFragment(); - /// Add something to the string table. - StringRef addToStringTable(StringRef S); - /// Get a string table offset. unsigned getStringTableOffset(StringRef S); - /// An array of absolute paths. Eventually this may include the file checksum. - SmallVector<StringRef, 4> Filenames; + struct FileInfo { + unsigned StringTableOffset; + + // Indicates if this FileInfo corresponds to an actual file, or hasn't been + // set yet. + bool Assigned = false; + + uint8_t ChecksumKind; + + ArrayRef<uint8_t> Checksum; + + // Checksum offset stored as a symbol because it might be requested + // before it has been calculated, so a fixup may be needed. + MCSymbol *ChecksumTableOffset; + }; + + /// Array storing added file information. + SmallVector<FileInfo, 4> Files; /// The offset of the first and last .cv_loc directive for a given function /// id. @@ -305,6 +325,10 @@ private: /// All known functions and inlined call sites, indexed by function id. std::vector<MCCVFunctionInfo> Functions; + + /// Indicate whether we have already laid out the checksum table addresses or + /// not. + bool ChecksumOffsetsAssigned = false; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCContext.h b/contrib/llvm/include/llvm/MC/MCContext.h index 2c60014adf23..432fc0ede072 100644 --- a/contrib/llvm/include/llvm/MC/MCContext.h +++ b/contrib/llvm/include/llvm/MC/MCContext.h @@ -441,53 +441,27 @@ namespace llvm { getAssociativeCOFFSection(MCSectionCOFF *Sec, const MCSymbol *KeySym, unsigned UniqueID = GenericSectionID); - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags) { - return getWasmSection(Section, Type, Flags, nullptr); + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K) { + return getWasmSection(Section, K, nullptr); } - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const char *BeginSymName) { - return getWasmSection(Section, Type, Flags, "", BeginSymName); - } - - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const Twine &Group) { - return getWasmSection(Section, Type, Flags, Group, nullptr); - } - - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const Twine &Group, + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, const char *BeginSymName) { - return getWasmSection(Section, Type, Flags, Group, ~0, BeginSymName); + return getWasmSection(Section, K, "", ~0, BeginSymName); } - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const Twine &Group, - unsigned UniqueID) { - return getWasmSection(Section, Type, Flags, Group, UniqueID, nullptr); + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + const Twine &Group, unsigned UniqueID) { + return getWasmSection(Section, K, Group, UniqueID, nullptr); } - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const Twine &Group, - unsigned UniqueID, const char *BeginSymName); - - MCSectionWasm *getWasmSection(const Twine &Section, unsigned Type, - unsigned Flags, const MCSymbolWasm *Group, - unsigned UniqueID, const char *BeginSymName); - - /// Get a section with the provided group identifier. This section is - /// named by concatenating \p Prefix with '.' then \p Suffix. The \p Type - /// describes the type of the section and \p Flags are used to further - /// configure this named section. - MCSectionWasm *getWasmNamedSection(const Twine &Prefix, const Twine &Suffix, - unsigned Type, unsigned Flags); - - MCSectionWasm *createWasmRelSection(const Twine &Name, unsigned Type, - unsigned Flags, - const MCSymbolWasm *Group); + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + const Twine &Group, unsigned UniqueID, + const char *BeginSymName); - void renameWasmSection(MCSectionWasm *Section, StringRef Name); + MCSectionWasm *getWasmSection(const Twine &Section, SectionKind K, + const MCSymbolWasm *Group, unsigned UniqueID, + const char *BeginSymName); // Create and save a copy of STI and return a reference to the copy. MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI); diff --git a/contrib/llvm/include/llvm/MC/MCDwarf.h b/contrib/llvm/include/llvm/MC/MCDwarf.h index 79f1b9525019..88ffa04128e6 100644 --- a/contrib/llvm/include/llvm/MC/MCDwarf.h +++ b/contrib/llvm/include/llvm/MC/MCDwarf.h @@ -509,6 +509,7 @@ struct MCDwarfFrameInfo { uint32_t CompactUnwindEncoding = 0; bool IsSignalFrame = false; bool IsSimple = false; + unsigned RAReg = static_cast<unsigned>(INT_MAX); }; class MCDwarfFrameEmitter { diff --git a/contrib/llvm/include/llvm/MC/MCELFObjectWriter.h b/contrib/llvm/include/llvm/MC/MCELFObjectWriter.h index 2efd37924e2e..fd8d118ccdc5 100644 --- a/contrib/llvm/include/llvm/MC/MCELFObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCELFObjectWriter.h @@ -55,11 +55,10 @@ class MCELFObjectTargetWriter { const uint16_t EMachine; const unsigned HasRelocationAddend : 1; const unsigned Is64Bit : 1; - const unsigned IsN64 : 1; protected: MCELFObjectTargetWriter(bool Is64Bit_, uint8_t OSABI_, uint16_t EMachine_, - bool HasRelocationAddend, bool IsN64 = false); + bool HasRelocationAddend); public: virtual ~MCELFObjectTargetWriter() = default; @@ -91,7 +90,6 @@ public: uint16_t getEMachine() const { return EMachine; } bool hasRelocationAddend() const { return HasRelocationAddend; } bool is64Bit() const { return Is64Bit; } - bool isN64() const { return IsN64; } /// @} // Instead of changing everyone's API we pack the N64 Type fields @@ -139,9 +137,9 @@ public: /// \param MOTW - The target specific ELF writer subclass. /// \param OS - The stream to write to. /// \returns The constructed object writer. -MCObjectWriter *createELFObjectWriter(MCELFObjectTargetWriter *MOTW, - raw_pwrite_stream &OS, - bool IsLittleEndian); +std::unique_ptr<MCObjectWriter> +createELFObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW, + raw_pwrite_stream &OS, bool IsLittleEndian); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCELFStreamer.h b/contrib/llvm/include/llvm/MC/MCELFStreamer.h index 90434f34da5f..c5b66a163c85 100644 --- a/contrib/llvm/include/llvm/MC/MCELFStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCELFStreamer.h @@ -23,9 +23,8 @@ class MCInst; class MCELFStreamer : public MCObjectStreamer { public: - MCELFStreamer(MCContext &Context, MCAsmBackend &TAB, raw_pwrite_stream &OS, - MCCodeEmitter *Emitter) - : MCObjectStreamer(Context, TAB, OS, Emitter) {} + MCELFStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, + raw_pwrite_stream &OS, std::unique_ptr<MCCodeEmitter> Emitter); ~MCELFStreamer() override = default; @@ -90,10 +89,11 @@ private: SmallVector<MCDataFragment *, 4> BundleGroups; }; -MCELFStreamer *createARMELFStreamer(MCContext &Context, MCAsmBackend &TAB, +MCELFStreamer *createARMELFStreamer(MCContext &Context, + std::unique_ptr<MCAsmBackend> TAB, raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, bool RelaxAll, - bool IsThumb); + std::unique_ptr<MCCodeEmitter> Emitter, + bool RelaxAll, bool IsThumb); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCExpr.h b/contrib/llvm/include/llvm/MC/MCExpr.h index a91a31414bdb..fcbbe650d26f 100644 --- a/contrib/llvm/include/llvm/MC/MCExpr.h +++ b/contrib/llvm/include/llvm/MC/MCExpr.h @@ -206,6 +206,14 @@ public: VK_ARM_TLSLDO, // symbol(tlsldo) VK_ARM_TLSDESCSEQ, + VK_AVR_NONE, + VK_AVR_LO8, + VK_AVR_HI8, + VK_AVR_HLO8, + VK_AVR_DIFF8, + VK_AVR_DIFF16, + VK_AVR_DIFF32, + VK_PPC_LO, // symbol@l VK_PPC_HI, // symbol@h VK_PPC_HA, // symbol@ha diff --git a/contrib/llvm/include/llvm/MC/MCFragment.h b/contrib/llvm/include/llvm/MC/MCFragment.h index 284ca50e19d5..7ebde03a758c 100644 --- a/contrib/llvm/include/llvm/MC/MCFragment.h +++ b/contrib/llvm/include/llvm/MC/MCFragment.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ilist_node.h" #include "llvm/MC/MCFixup.h" #include "llvm/MC/MCInst.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/SMLoc.h" #include <cstdint> #include <utility> @@ -41,7 +42,8 @@ public: FT_Dwarf, FT_DwarfFrame, FT_LEB, - FT_SafeSEH, + FT_Padding, + FT_SymbolId, FT_CVInlineLines, FT_CVDefRange, FT_Dummy @@ -323,6 +325,98 @@ public: } }; +/// Fragment for adding required padding. +/// This fragment is always inserted before an instruction, and holds that +/// instruction as context information (as well as a mask of kinds) for +/// determining the padding size. +/// +class MCPaddingFragment : public MCFragment { + /// A mask containing all the kinds relevant to this fragment. i.e. the i'th + /// bit will be set iff kind i is relevant to this fragment. + uint64_t PaddingPoliciesMask; + /// A boolean indicating if this fragment will actually hold padding. If its + /// value is false, then this fragment serves only as a placeholder, + /// containing data to assist other insertion point in their decision making. + bool IsInsertionPoint; + + uint64_t Size; + + struct MCInstInfo { + bool IsInitialized; + MCInst Inst; + /// A boolean indicating whether the instruction pointed by this fragment is + /// a fixed size instruction or a relaxable instruction held by a + /// MCRelaxableFragment. + bool IsImmutableSizedInst; + union { + /// If the instruction is a fixed size instruction, hold its size. + size_t InstSize; + /// Otherwise, hold a pointer to the MCRelaxableFragment holding it. + MCRelaxableFragment *InstFragment; + }; + }; + MCInstInfo InstInfo; + +public: + static const uint64_t PFK_None = UINT64_C(0); + + enum MCPaddingFragmentKind { + // values 0-7 are reserved for future target independet values. + + FirstTargetPerfNopFragmentKind = 8, + + /// Limit range of target MCPerfNopFragment kinds to fit in uint64_t + MaxTargetPerfNopFragmentKind = 63 + }; + + MCPaddingFragment(MCSection *Sec = nullptr) + : MCFragment(FT_Padding, false, 0, Sec), PaddingPoliciesMask(PFK_None), + IsInsertionPoint(false), Size(UINT64_C(0)), + InstInfo({false, MCInst(), false, {0}}) {} + + bool isInsertionPoint() const { return IsInsertionPoint; } + void setAsInsertionPoint() { IsInsertionPoint = true; } + uint64_t getPaddingPoliciesMask() const { return PaddingPoliciesMask; } + void setPaddingPoliciesMask(uint64_t Value) { PaddingPoliciesMask = Value; } + bool hasPaddingPolicy(uint64_t PolicyMask) const { + assert(isPowerOf2_64(PolicyMask) && + "Policy mask must contain exactly one policy"); + return (getPaddingPoliciesMask() & PolicyMask) != PFK_None; + } + const MCInst &getInst() const { + assert(isInstructionInitialized() && "Fragment has no instruction!"); + return InstInfo.Inst; + } + size_t getInstSize() const { + assert(isInstructionInitialized() && "Fragment has no instruction!"); + if (InstInfo.IsImmutableSizedInst) + return InstInfo.InstSize; + assert(InstInfo.InstFragment != nullptr && + "Must have a valid InstFragment to retrieve InstSize from"); + return InstInfo.InstFragment->getContents().size(); + } + void setInstAndInstSize(const MCInst &Inst, size_t InstSize) { + InstInfo.IsInitialized = true; + InstInfo.IsImmutableSizedInst = true; + InstInfo.Inst = Inst; + InstInfo.InstSize = InstSize; + } + void setInstAndInstFragment(const MCInst &Inst, + MCRelaxableFragment *InstFragment) { + InstInfo.IsInitialized = true; + InstInfo.IsImmutableSizedInst = false; + InstInfo.Inst = Inst; + InstInfo.InstFragment = InstFragment; + } + uint64_t getSize() const { return Size; } + void setSize(uint64_t Value) { Size = Value; } + bool isInstructionInitialized() const { return InstInfo.IsInitialized; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Padding; + } +}; + class MCFillFragment : public MCFragment { /// Value to use for filling bytes. uint8_t Value; @@ -469,12 +563,13 @@ public: } }; -class MCSafeSEHFragment : public MCFragment { +/// Represents a symbol table index fragment. +class MCSymbolIdFragment : public MCFragment { const MCSymbol *Sym; public: - MCSafeSEHFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) - : MCFragment(FT_SafeSEH, false, 0, Sec), Sym(Sym) {} + MCSymbolIdFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) + : MCFragment(FT_SymbolId, false, 0, Sec), Sym(Sym) {} /// \name Accessors /// @{ @@ -485,7 +580,7 @@ public: /// @} static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_SafeSEH; + return F->getKind() == MCFragment::FT_SymbolId; } }; diff --git a/contrib/llvm/include/llvm/MC/MCInst.h b/contrib/llvm/include/llvm/MC/MCInst.h index 9bf440ea96d2..db28fd0fd6d9 100644 --- a/contrib/llvm/include/llvm/MC/MCInst.h +++ b/contrib/llvm/include/llvm/MC/MCInst.h @@ -160,6 +160,10 @@ class MCInst { unsigned Opcode = 0; SMLoc Loc; SmallVector<MCOperand, 8> Operands; + // These flags could be used to pass some info from one target subcomponent + // to another, for example, from disassembler to asm printer. The values of + // the flags have any sense on target level only (e.g. prefixes on x86). + unsigned Flags = 0; public: MCInst() = default; @@ -167,6 +171,9 @@ public: void setOpcode(unsigned Op) { Opcode = Op; } unsigned getOpcode() const { return Opcode; } + void setFlags(unsigned F) { Flags = F; } + unsigned getFlags() const { return Flags; } + void setLoc(SMLoc loc) { Loc = loc; } SMLoc getLoc() const { return Loc; } diff --git a/contrib/llvm/include/llvm/MC/MCInstrDesc.h b/contrib/llvm/include/llvm/MC/MCInstrDesc.h index 9150a8b5c80a..ff4c756a66a1 100644 --- a/contrib/llvm/include/llvm/MC/MCInstrDesc.h +++ b/contrib/llvm/include/llvm/MC/MCInstrDesc.h @@ -173,7 +173,7 @@ public: const MCPhysReg *ImplicitDefs; // Registers implicitly defined by this instr const MCOperandInfo *OpInfo; // 'NumOperands' entries about operands // Subtarget feature that this is deprecated on, if any - // -1 implies this is not deprecated by any single feature. It may still be + // -1 implies this is not deprecated by any single feature. It may still be // deprecated due to a "complex" reason, below. int64_t DeprecatedFeature; @@ -580,8 +580,6 @@ public: return -1; } -private: - /// \brief Return true if this instruction defines the specified physical /// register, either explicitly or implicitly. bool hasDefOfPhysReg(const MCInst &MI, unsigned Reg, diff --git a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h index 42dc90da3049..594869f74632 100644 --- a/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -117,9 +117,10 @@ class MachObjectWriter : public MCObjectWriter { MachSymbolData *findSymbolData(const MCSymbol &Sym); public: - MachObjectWriter(MCMachObjectTargetWriter *MOTW, raw_pwrite_stream &OS, - bool IsLittleEndian) - : MCObjectWriter(OS, IsLittleEndian), TargetObjectWriter(MOTW) {} + MachObjectWriter(std::unique_ptr<MCMachObjectTargetWriter> MOTW, + raw_pwrite_stream &OS, bool IsLittleEndian) + : MCObjectWriter(OS, IsLittleEndian), + TargetObjectWriter(std::move(MOTW)) {} const MCSymbol &findAliasedSymbol(const MCSymbol &Sym) const; @@ -269,9 +270,9 @@ public: /// \param MOTW - The target specific Mach-O writer subclass. /// \param OS - The stream to write to. /// \returns The constructed object writer. -MCObjectWriter *createMachObjectWriter(MCMachObjectTargetWriter *MOTW, - raw_pwrite_stream &OS, - bool IsLittleEndian); +std::unique_ptr<MCObjectWriter> +createMachObjectWriter(std::unique_ptr<MCMachObjectTargetWriter> MOTW, + raw_pwrite_stream &OS, bool IsLittleEndian); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h index 4d634447987b..79bf2b97015f 100644 --- a/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/contrib/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -123,8 +123,12 @@ protected: /// Section for newer gnu pubtypes. MCSection *DwarfGnuPubTypesSection; + // Section for Swift AST + MCSection *DwarfSwiftASTSection; + MCSection *COFFDebugSymbolsSection; MCSection *COFFDebugTypesSection; + MCSection *COFFGlobalTypeHashesSection; /// Extra TLS Variable Data section. /// @@ -151,6 +155,9 @@ protected: /// It is initialized on demand so it can be overwritten (with uniquing). MCSection *EHFrameSection; + /// Section containing metadata on function stack sizes. + MCSection *StackSizesSection; + // ELF specific sections. MCSection *DataRelROSection; MCSection *MergeableConst4Section; @@ -191,8 +198,8 @@ protected: MCSection *SXDataSection; public: - void InitMCObjectFileInfo(const Triple &TT, bool PIC, CodeModel::Model CM, - MCContext &ctx); + void InitMCObjectFileInfo(const Triple &TT, bool PIC, MCContext &ctx, + bool LargeCodeModel = false); bool getSupportsWeakOmittedEHFrame() const { return SupportsWeakOmittedEHFrame; @@ -267,6 +274,7 @@ public: MCSection *getDwarfAddrSection() const { return DwarfAddrSection; } MCSection *getDwarfCUIndexSection() const { return DwarfCUIndexSection; } MCSection *getDwarfTUIndexSection() const { return DwarfTUIndexSection; } + MCSection *getDwarfSwiftASTSection() const { return DwarfSwiftASTSection; } MCSection *getCOFFDebugSymbolsSection() const { return COFFDebugSymbolsSection; @@ -274,7 +282,9 @@ public: MCSection *getCOFFDebugTypesSection() const { return COFFDebugTypesSection; } - + MCSection *getCOFFGlobalTypeHashesSection() const { + return COFFGlobalTypeHashesSection; + } MCSection *getTLSExtraDataSection() const { return TLSExtraDataSection; } const MCSection *getTLSDataSection() const { return TLSDataSection; } @@ -283,6 +293,8 @@ public: MCSection *getStackMapSection() const { return StackMapSection; } MCSection *getFaultMapSection() const { return FaultMapSection; } + MCSection *getStackSizesSection() const { return StackSizesSection; } + // ELF specific sections. MCSection *getDataRelROSection() const { return DataRelROSection; } const MCSection *getMergeableConst4Section() const { @@ -350,12 +362,11 @@ public: private: Environment Env; bool PositionIndependent; - CodeModel::Model CMModel; MCContext *Ctx; Triple TT; void initMachOMCObjectFileInfo(const Triple &T); - void initELFMCObjectFileInfo(const Triple &T); + void initELFMCObjectFileInfo(const Triple &T, bool Large); void initCOFFMCObjectFileInfo(const Triple &T); void initWasmMCObjectFileInfo(const Triple &T); diff --git a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h index 7c1189e46ab2..a3dbc56ebc10 100644 --- a/contrib/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCObjectStreamer.h @@ -34,7 +34,10 @@ class raw_pwrite_stream; /// to that file format or custom semantics expected by the object writer /// implementation. class MCObjectStreamer : public MCStreamer { - MCAssembler *Assembler; + std::unique_ptr<MCObjectWriter> ObjectWriter; + std::unique_ptr<MCAsmBackend> TAB; + std::unique_ptr<MCCodeEmitter> Emitter; + std::unique_ptr<MCAssembler> Assembler; MCSection::iterator CurInsertionPoint; bool EmitEHFrame; bool EmitDebugFrame; @@ -43,11 +46,14 @@ class MCObjectStreamer : public MCStreamer { virtual void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0; void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; void EmitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; + MCSymbol *EmitCFILabel() override; + void EmitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI); protected: - MCObjectStreamer(MCContext &Context, MCAsmBackend &TAB, raw_pwrite_stream &OS, - MCCodeEmitter *Emitter); - ~MCObjectStreamer() override; + MCObjectStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> Emitter); + ~MCObjectStreamer(); public: /// state management @@ -71,6 +77,7 @@ public: /// Get a data fragment to write into, creating a new one if the current /// fragment is not a data fragment. MCDataFragment *getOrCreateDataFragment(); + MCPaddingFragment *getOrCreatePaddingFragment(); protected: bool changeSectionImpl(MCSection *Section, const MCExpr *Subsection); @@ -116,6 +123,10 @@ public: unsigned MaxBytesToEmit = 0) override; void emitValueToOffset(const MCExpr *Offset, unsigned char Value, SMLoc Loc) override; + void + EmitCodePaddingBasicBlockStart(const MCCodePaddingContext &Context) override; + void + EmitCodePaddingBasicBlockEnd(const MCCodePaddingContext &Context) override; void EmitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column, unsigned Flags, unsigned Isa, unsigned Discriminator, @@ -140,6 +151,7 @@ public: StringRef FixedSizePortion) override; void EmitCVStringTableDirective() override; void EmitCVFileChecksumsDirective() override; + void EmitCVFileChecksumOffsetDirective(unsigned FileNo) override; void EmitDTPRel32Value(const MCExpr *Value) override; void EmitDTPRel64Value(const MCExpr *Value) override; void EmitTPRel32Value(const MCExpr *Value) override; diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h index 3a659f048ccf..0f79c4777ea9 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/contrib/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -34,19 +34,61 @@ class MCStreamer; class MCTargetAsmParser; class SourceMgr; -class InlineAsmIdentifierInfo { -public: - void *OpDecl; - bool IsVarDecl; - unsigned Length, Size, Type; - - void clear() { - OpDecl = nullptr; - IsVarDecl = false; - Length = 1; - Size = 0; - Type = 0; +struct InlineAsmIdentifierInfo { + enum IdKind { + IK_Invalid, // Initial state. Unexpected after a successful parsing. + IK_Label, // Function/Label reference. + IK_EnumVal, // Value of enumeration type. + IK_Var // Variable. + }; + // Represents an Enum value + struct EnumIdentifier { + int64_t EnumVal; + }; + // Represents a label/function reference + struct LabelIdentifier { + void *Decl; + }; + // Represents a variable + struct VariableIdentifier { + void *Decl; + bool IsGlobalLV; + unsigned Length; + unsigned Size; + unsigned Type; + }; + // An InlineAsm identifier can only be one of those + union { + EnumIdentifier Enum; + LabelIdentifier Label; + VariableIdentifier Var; + }; + bool isKind(IdKind kind) const { return Kind == kind; } + // Initializers + void setEnum(int64_t enumVal) { + assert(isKind(IK_Invalid) && "should be initialized only once"); + Kind = IK_EnumVal; + Enum.EnumVal = enumVal; + } + void setLabel(void *decl) { + assert(isKind(IK_Invalid) && "should be initialized only once"); + Kind = IK_Label; + Label.Decl = decl; } + void setVar(void *decl, bool isGlobalLV, unsigned size, unsigned type) { + assert(isKind(IK_Invalid) && "should be initialized only once"); + Kind = IK_Var; + Var.Decl = decl; + Var.IsGlobalLV = isGlobalLV; + Var.Size = size; + Var.Type = type; + Var.Length = size / type; + } + InlineAsmIdentifierInfo() : Kind(IK_Invalid) {} + +private: + // Discriminate using the current kind. + IdKind Kind; }; /// \brief Generic Sema callback for assembly parser. @@ -54,9 +96,9 @@ class MCAsmParserSemaCallback { public: virtual ~MCAsmParserSemaCallback(); - virtual void *LookupInlineAsmIdentifier(StringRef &LineBuf, - InlineAsmIdentifierInfo &Info, - bool IsUnevaluatedContext) = 0; + virtual void LookupInlineAsmIdentifier(StringRef &LineBuf, + InlineAsmIdentifierInfo &Info, + bool IsUnevaluatedContext) = 0; virtual StringRef LookupInlineAsmLabel(StringRef Identifier, SourceMgr &SM, SMLoc Location, bool Create) = 0; virtual bool LookupInlineAsmField(StringRef Base, StringRef Member, @@ -85,10 +127,12 @@ private: protected: // Can only create subclasses. MCAsmParser(); + /// Flag tracking whether any errors have been encountered. bool HadError = false; + /// Enable print [latency:throughput] in output file. + bool EnablePrintSchedInfo = false; SmallVector<MCPendingError, 1> PendingErrors; - /// Flag tracking whether any errors have been encountered. public: MCAsmParser(const MCAsmParser &) = delete; @@ -121,6 +165,9 @@ public: bool getShowParsedOperands() const { return ShowParsedOperands; } void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; } + void setEnablePrintSchedInfo(bool Value) { EnablePrintSchedInfo = Value; } + bool shouldPrintSchedInfo() { return EnablePrintSchedInfo; } + /// \brief Run the parser on the input source buffer. virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false) = 0; diff --git a/contrib/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/contrib/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index b8d3180cd49c..9f8550c3887c 100644 --- a/contrib/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/contrib/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -12,6 +12,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCParser/MCAsmLexer.h" #include "llvm/MC/MCParser/MCAsmParserExtension.h" #include "llvm/MC/MCTargetOptions.h" @@ -30,50 +31,92 @@ template <typename T> class SmallVectorImpl; using OperandVector = SmallVectorImpl<std::unique_ptr<MCParsedAsmOperand>>; enum AsmRewriteKind { - AOK_Delete = 0, // Rewrite should be ignored. AOK_Align, // Rewrite align as .align. AOK_EVEN, // Rewrite even as .even. - AOK_DotOperator, // Rewrite a dot operator expression as an immediate. - // E.g., [eax].foo.bar -> [eax].8 AOK_Emit, // Rewrite _emit as .byte. - AOK_Imm, // Rewrite as $$N. - AOK_ImmPrefix, // Add $$ before a parsed Imm. AOK_Input, // Rewrite in terms of $N. AOK_Output, // Rewrite in terms of $N. AOK_SizeDirective, // Add a sizing directive (e.g., dword ptr). AOK_Label, // Rewrite local labels. AOK_EndOfStatement, // Add EndOfStatement (e.g., "\n\t"). - AOK_Skip // Skip emission (e.g., offset/type operators). + AOK_Skip, // Skip emission (e.g., offset/type operators). + AOK_IntelExpr // SizeDirective SymDisp [BaseReg + IndexReg * Scale + ImmDisp] }; const char AsmRewritePrecedence [] = { - 0, // AOK_Delete 2, // AOK_Align 2, // AOK_EVEN - 2, // AOK_DotOperator 2, // AOK_Emit - 4, // AOK_Imm - 4, // AOK_ImmPrefix 3, // AOK_Input 3, // AOK_Output 5, // AOK_SizeDirective 1, // AOK_Label 5, // AOK_EndOfStatement - 2 // AOK_Skip + 2, // AOK_Skip + 2 // AOK_IntelExpr +}; + +// Represnt the various parts which makes up an intel expression, +// used for emitting compound intel expressions +struct IntelExpr { + bool NeedBracs; + int64_t Imm; + StringRef BaseReg; + StringRef IndexReg; + unsigned Scale; + + IntelExpr(bool needBracs = false) : NeedBracs(needBracs), Imm(0), + BaseReg(StringRef()), IndexReg(StringRef()), + Scale(1) {} + // Compund immediate expression + IntelExpr(int64_t imm, bool needBracs) : IntelExpr(needBracs) { + Imm = imm; + } + // [Reg + ImmediateExpression] + // We don't bother to emit an immediate expression evaluated to zero + IntelExpr(StringRef reg, int64_t imm = 0, unsigned scale = 0, + bool needBracs = true) : + IntelExpr(imm, needBracs) { + IndexReg = reg; + if (scale) + Scale = scale; + } + // [BaseReg + IndexReg * ScaleExpression + ImmediateExpression] + IntelExpr(StringRef baseReg, StringRef indexReg, unsigned scale = 0, + int64_t imm = 0, bool needBracs = true) : + IntelExpr(indexReg, imm, scale, needBracs) { + BaseReg = baseReg; + } + bool hasBaseReg() const { + return BaseReg.size(); + } + bool hasIndexReg() const { + return IndexReg.size(); + } + bool hasRegs() const { + return hasBaseReg() || hasIndexReg(); + } + bool isValid() const { + return (Scale == 1) || + (hasIndexReg() && (Scale == 2 || Scale == 4 || Scale == 8)); + } }; struct AsmRewrite { AsmRewriteKind Kind; SMLoc Loc; unsigned Len; - unsigned Val; + int64_t Val; StringRef Label; + IntelExpr IntelExp; public: - AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, unsigned val = 0) + AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, int64_t val = 0) : Kind(kind), Loc(loc), Len(len), Val(val) {} AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len, StringRef label) - : Kind(kind), Loc(loc), Len(len), Val(0), Label(label) {} + : AsmRewrite(kind, loc, len) { Label = label; } + AsmRewrite(SMLoc loc, unsigned len, IntelExpr exp) + : AsmRewrite(AOK_IntelExpr, loc, len) { IntelExp = exp; } }; struct ParseInstructionInfo { @@ -90,6 +133,139 @@ enum OperandMatchResultTy { MatchOperand_ParseFail // operand matched but had errors }; +// When matching of an assembly instruction fails, there may be multiple +// encodings that are close to being a match. It's often ambiguous which one +// the programmer intended to use, so we want to report an error which mentions +// each of these "near-miss" encodings. This struct contains information about +// one such encoding, and why it did not match the parsed instruction. +class NearMissInfo { +public: + enum NearMissKind { + NoNearMiss, + NearMissOperand, + NearMissFeature, + NearMissPredicate, + NearMissTooFewOperands, + }; + + // The encoding is valid for the parsed assembly string. This is only used + // internally to the table-generated assembly matcher. + static NearMissInfo getSuccess() { return NearMissInfo(); } + + // The instruction encoding is not valid because it requires some target + // features that are not currently enabled. MissingFeatures has a bit set for + // each feature that the encoding needs but which is not enabled. + static NearMissInfo getMissedFeature(uint64_t MissingFeatures) { + NearMissInfo Result; + Result.Kind = NearMissFeature; + Result.Features = MissingFeatures; + return Result; + } + + // The instruction encoding is not valid because the target-specific + // predicate function returned an error code. FailureCode is the + // target-specific error code returned by the predicate. + static NearMissInfo getMissedPredicate(unsigned FailureCode) { + NearMissInfo Result; + Result.Kind = NearMissPredicate; + Result.PredicateError = FailureCode; + return Result; + } + + // The instruction encoding is not valid because one (and only one) parsed + // operand is not of the correct type. OperandError is the error code + // relating to the operand class expected by the encoding. OperandClass is + // the type of the expected operand. Opcode is the opcode of the encoding. + // OperandIndex is the index into the parsed operand list. + static NearMissInfo getMissedOperand(unsigned OperandError, + unsigned OperandClass, unsigned Opcode, + unsigned OperandIndex) { + NearMissInfo Result; + Result.Kind = NearMissOperand; + Result.MissedOperand.Error = OperandError; + Result.MissedOperand.Class = OperandClass; + Result.MissedOperand.Opcode = Opcode; + Result.MissedOperand.Index = OperandIndex; + return Result; + } + + // The instruction encoding is not valid because it expects more operands + // than were parsed. OperandClass is the class of the expected operand that + // was not provided. Opcode is the instruction encoding. + static NearMissInfo getTooFewOperands(unsigned OperandClass, + unsigned Opcode) { + NearMissInfo Result; + Result.Kind = NearMissTooFewOperands; + Result.TooFewOperands.Class = OperandClass; + Result.TooFewOperands.Opcode = Opcode; + return Result; + } + + operator bool() const { return Kind != NoNearMiss; } + + NearMissKind getKind() const { return Kind; } + + // Feature flags required by the instruction, that the current target does + // not have. + uint64_t getFeatures() const { + assert(Kind == NearMissFeature); + return Features; + } + // Error code returned by the target predicate when validating this + // instruction encoding. + unsigned getPredicateError() const { + assert(Kind == NearMissPredicate); + return PredicateError; + } + // MatchClassKind of the operand that we expected to see. + unsigned getOperandClass() const { + assert(Kind == NearMissOperand || Kind == NearMissTooFewOperands); + return MissedOperand.Class; + } + // Opcode of the encoding we were trying to match. + unsigned getOpcode() const { + assert(Kind == NearMissOperand || Kind == NearMissTooFewOperands); + return MissedOperand.Opcode; + } + // Error code returned when validating the operand. + unsigned getOperandError() const { + assert(Kind == NearMissOperand); + return MissedOperand.Error; + } + // Index of the actual operand we were trying to match in the list of parsed + // operands. + unsigned getOperandIndex() const { + assert(Kind == NearMissOperand); + return MissedOperand.Index; + } + +private: + NearMissKind Kind; + + // These two structs share a common prefix, so we can safely rely on the fact + // that they overlap in the union. + struct MissedOpInfo { + unsigned Class; + unsigned Opcode; + unsigned Error; + unsigned Index; + }; + + struct TooFewOperandsInfo { + unsigned Class; + unsigned Opcode; + }; + + union { + uint64_t Features; + unsigned PredicateError; + MissedOpInfo MissedOperand; + TooFewOperandsInfo TooFewOperands; + }; + + NearMissInfo() : Kind(NoNearMiss) {} +}; + /// MCTargetAsmParser - Generic interface to target specific assembly parsers. class MCTargetAsmParser : public MCAsmParserExtension { public: @@ -98,11 +274,13 @@ public: Match_MissingFeature, Match_MnemonicFail, Match_Success, + Match_NearMisses, FIRST_TARGET_MATCH_RESULT_TY }; protected: // Can only create subclasses. - MCTargetAsmParser(MCTargetOptions const &, const MCSubtargetInfo &STI); + MCTargetAsmParser(MCTargetOptions const &, const MCSubtargetInfo &STI, + const MCInstrInfo &MII); /// Create a copy of STI and return a non-const reference to it. MCSubtargetInfo ©STI(); @@ -123,6 +301,8 @@ protected: // Can only create subclasses. /// Current STI. const MCSubtargetInfo *STI; + const MCInstrInfo &MII; + public: MCTargetAsmParser(const MCTargetAsmParser &) = delete; MCTargetAsmParser &operator=(const MCTargetAsmParser &) = delete; @@ -224,6 +404,8 @@ public: virtual bool equalIsAsmAssignment() { return true; }; // Return whether this start of statement identifier is a label virtual bool isLabel(AsmToken &Token) { return true; }; + // Return whether this parser accept star as start of statement + virtual bool starIsStartOfStatement() { return false; }; virtual const MCExpr *applyModifierToExpr(const MCExpr *E, MCSymbolRefExpr::VariantKind, diff --git a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h index de98abe0dc46..c57c9ef709da 100644 --- a/contrib/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/contrib/llvm/include/llvm/MC/MCRegisterInfo.h @@ -407,6 +407,15 @@ public: /// \brief Map a dwarf register back to a target register. int getLLVMRegNum(unsigned RegNum, bool isEH) const; + /// \brief Map a DWARF EH register back to a target register (same as + /// getLLVMRegNum(RegNum, true)) but return -1 if there is no mapping, + /// rather than asserting that there must be one. + int getLLVMRegNumFromEH(unsigned RegNum) const; + + /// \brief Map a target EH register number to an equivalent DWARF register + /// number. + int getDwarfRegNumFromDwarfEHRegNum(unsigned RegNum) const; + /// \brief Map a target register to an equivalent SEH register /// number. Returns LLVM register number if there is no equivalent value. int getSEHRegNum(unsigned RegNum) const; diff --git a/contrib/llvm/include/llvm/MC/MCSchedule.h b/contrib/llvm/include/llvm/MC/MCSchedule.h index 37728797f626..a79afe163e6c 100644 --- a/contrib/llvm/include/llvm/MC/MCSchedule.h +++ b/contrib/llvm/include/llvm/MC/MCSchedule.h @@ -24,7 +24,7 @@ struct InstrItinerary; /// Define a kind of processor resource that will be modeled by the scheduler. struct MCProcResourceDesc { -#ifndef NDEBUG +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) const char *Name; #endif unsigned NumUnits; // Number of resource of this kind @@ -102,7 +102,7 @@ struct MCSchedClassDesc { static const unsigned short InvalidNumMicroOps = UINT16_MAX; static const unsigned short VariantNumMicroOps = UINT16_MAX - 1; -#ifndef NDEBUG +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) const char* Name; #endif unsigned short NumMicroOps; diff --git a/contrib/llvm/include/llvm/MC/MCSectionWasm.h b/contrib/llvm/include/llvm/MC/MCSectionWasm.h index 29d62a7a6f82..cc467ed9837a 100644 --- a/contrib/llvm/include/llvm/MC/MCSectionWasm.h +++ b/contrib/llvm/include/llvm/MC/MCSectionWasm.h @@ -27,16 +27,11 @@ class MCSymbol; /// This represents a section on wasm. class MCSectionWasm final : public MCSection { private: + /// This is the name of the section. The referenced memory is owned by /// TargetLoweringObjectFileWasm's WasmUniqueMap. StringRef SectionName; - /// This is the sh_type field of a section, drawn from the enums below. - unsigned Type; - - /// This is the sh_flags field of a section, drawn from the enums below. - unsigned Flags; - unsigned UniqueID; const MCSymbolWasm *Group; @@ -46,12 +41,15 @@ private: // itself and does not include the size of the section header. uint64_t SectionOffset; + // For data sections, this is the offset of the corresponding wasm data + // segment + uint64_t MemoryOffset; + friend class MCContext; - MCSectionWasm(StringRef Section, unsigned type, unsigned flags, SectionKind K, - const MCSymbolWasm *group, unsigned UniqueID, MCSymbol *Begin) - : MCSection(SV_Wasm, K, Begin), SectionName(Section), Type(type), - Flags(flags), UniqueID(UniqueID), Group(group), SectionOffset(0) { - } + MCSectionWasm(StringRef Section, SectionKind K, const MCSymbolWasm *group, + unsigned UniqueID, MCSymbol *Begin) + : MCSection(SV_Wasm, K, Begin), SectionName(Section), UniqueID(UniqueID), + Group(group), SectionOffset(0) {} void setSectionName(StringRef Name) { SectionName = Name; } @@ -63,9 +61,6 @@ public: bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; StringRef getSectionName() const { return SectionName; } - unsigned getType() const { return Type; } - unsigned getFlags() const { return Flags; } - void setFlags(unsigned F) { Flags = F; } const MCSymbolWasm *getGroup() const { return Group; } void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, @@ -74,12 +69,19 @@ public: bool UseCodeAlign() const override; bool isVirtualSection() const override; + bool isWasmData() const { + return Kind.isGlobalWriteableData() || Kind.isReadOnly(); + } + bool isUnique() const { return UniqueID != ~0U; } unsigned getUniqueID() const { return UniqueID; } uint64_t getSectionOffset() const { return SectionOffset; } void setSectionOffset(uint64_t Offset) { SectionOffset = Offset; } + uint32_t getMemoryOffset() const { return MemoryOffset; } + void setMemoryOffset(uint32_t Offset) { MemoryOffset = Offset; } + static bool classof(const MCSection *S) { return S->getVariant() == SV_Wasm; } }; diff --git a/contrib/llvm/include/llvm/MC/MCStreamer.h b/contrib/llvm/include/llvm/MC/MCStreamer.h index 5390e7942424..481d96724d40 100644 --- a/contrib/llvm/include/llvm/MC/MCStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCStreamer.h @@ -24,6 +24,7 @@ #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" #include "llvm/Support/SMLoc.h" +#include "llvm/Support/TargetParser.h" #include <cassert> #include <cstdint> #include <memory> @@ -37,6 +38,7 @@ class AssemblerConstantPools; class formatted_raw_ostream; class MCAsmBackend; class MCCodeEmitter; +struct MCCodePaddingContext; class MCContext; class MCExpr; class MCInst; @@ -124,9 +126,9 @@ public: virtual void emitIntTextAttribute(unsigned Attribute, unsigned IntValue, StringRef StringValue = ""); virtual void emitFPU(unsigned FPU); - virtual void emitArch(unsigned Arch); + virtual void emitArch(ARM::ArchKind Arch); virtual void emitArchExtension(unsigned ArchExt); - virtual void emitObjectArch(unsigned Arch); + virtual void emitObjectArch(ARM::ArchKind Arch); void emitTargetAttributes(const MCSubtargetInfo &STI); virtual void finishAttributeSection(); virtual void emitInst(uint32_t Inst, char Suffix = '\0'); @@ -170,14 +172,16 @@ class MCStreamer { std::vector<MCDwarfFrameInfo> DwarfFrameInfos; MCDwarfFrameInfo *getCurrentDwarfFrameInfo(); - void EnsureValidDwarfFrame(); - MCSymbol *EmitCFILabel(); - MCSymbol *EmitCFICommon(); + /// Similar to DwarfFrameInfos, but for SEH unwind info. Chained frames may + /// refer to each other, so use std::unique_ptr to provide pointer stability. + std::vector<std::unique_ptr<WinEH::FrameInfo>> WinFrameInfos; - std::vector<WinEH::FrameInfo *> WinFrameInfos; WinEH::FrameInfo *CurrentWinFrameInfo; - void EnsureValidWinFrameInfo(); + + /// Retreive the current frame info if one is available and it is not yet + /// closed. Otherwise, issue an error and return null. + WinEH::FrameInfo *EnsureValidWinFrameInfo(SMLoc Loc); /// \brief Tracks an index to represent the order a symbol was emitted in. /// Zero means we did not emit that symbol. @@ -199,6 +203,10 @@ protected: virtual void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame); virtual void EmitCFIEndProcImpl(MCDwarfFrameInfo &CurFrame); + /// When emitting an object file, create and emit a real label. When emitting + /// textual assembly, this should do nothing to avoid polluting our output. + virtual MCSymbol *EmitCFILabel(); + WinEH::FrameInfo *getCurrentWinFrameInfo() { return CurrentWinFrameInfo; } @@ -237,7 +245,7 @@ public: bool hasUnfinishedDwarfFrameInfo(); unsigned getNumWinFrameInfos() { return WinFrameInfos.size(); } - ArrayRef<WinEH::FrameInfo *> getWinFrameInfos() const { + ArrayRef<std::unique_ptr<WinEH::FrameInfo>> getWinFrameInfos() const { return WinFrameInfos; } @@ -413,9 +421,16 @@ public: /// \brief Note in the output the specified region \p Kind. virtual void EmitDataRegion(MCDataRegionType Kind) {} - /// \brief Specify the MachO minimum deployment target version. - virtual void EmitVersionMin(MCVersionMinType, unsigned Major, unsigned Minor, - unsigned Update) {} + /// \brief Specify the Mach-O minimum deployment target version. + virtual void EmitVersionMin(MCVersionMinType Type, unsigned Major, + unsigned Minor, unsigned Update) {} + + /// Emit/Specify Mach-O build version command. + /// \p Platform should be one of MachO::PlatformType. + virtual void EmitBuildVersion(unsigned Platform, unsigned Major, + unsigned Minor, unsigned Update) {} + + void EmitVersionForTarget(const Triple &Target); /// \brief Note in the output that the specified \p Func is a Thumb mode /// function (ARM target only). @@ -577,7 +592,11 @@ public: /// \brief Special case of EmitULEB128Value that avoids the client having to /// pass in a MCExpr for constant integers. - void EmitULEB128IntValue(uint64_t Value, unsigned Padding = 0); + void EmitULEB128IntValue(uint64_t Value); + + /// \brief Like EmitULEB128Value but pads the output to specific number of + /// bytes. + void EmitPaddedULEB128IntValue(uint64_t Value, unsigned PadTo); /// \brief Special case of EmitSLEB128Value that avoids the client having to /// pass in a MCExpr for constant integers. @@ -705,6 +724,12 @@ public: virtual void emitValueToOffset(const MCExpr *Offset, unsigned char Value, SMLoc Loc); + virtual void + EmitCodePaddingBasicBlockStart(const MCCodePaddingContext &Context) {} + + virtual void + EmitCodePaddingBasicBlockEnd(const MCCodePaddingContext &Context) {} + /// @} /// \brief Switch to a new logical file. This is used to implement the '.file @@ -728,10 +753,12 @@ public: unsigned Isa, unsigned Discriminator, StringRef FileName); - /// \brief Associate a filename with a specified logical file number. This - /// implements the '.cv_file 4 "foo.c"' assembler directive. Returns true on - /// success. - virtual bool EmitCVFileDirective(unsigned FileNo, StringRef Filename); + /// Associate a filename with a specified logical file number, and also + /// specify that file's checksum information. This implements the '.cv_file 4 + /// "foo.c"' assembler directive. Returns true on success. + virtual bool EmitCVFileDirective(unsigned FileNo, StringRef Filename, + ArrayRef<uint8_t> Checksum, + unsigned ChecksumKind); /// \brief Introduces a function id for use with .cv_loc. virtual bool EmitCVFuncIdDirective(unsigned FunctionId); @@ -773,6 +800,13 @@ public: /// \brief This implements the CodeView '.cv_filechecksums' assembler directive. virtual void EmitCVFileChecksumsDirective() {} + /// This implements the CodeView '.cv_filechecksumoffset' assembler + /// directive. + virtual void EmitCVFileChecksumOffsetDirective(unsigned FileNo) {} + + /// This implements the CodeView '.cv_fpo_data' assembler directive. + virtual void EmitCVFPOData(const MCSymbol *ProcSym, SMLoc Loc = {}) {} + /// Emit the absolute difference between two symbols. /// /// \pre Offset of \c Hi is greater than the offset \c Lo. @@ -796,26 +830,30 @@ public: virtual void EmitCFIRelOffset(int64_t Register, int64_t Offset); virtual void EmitCFIAdjustCfaOffset(int64_t Adjustment); virtual void EmitCFIEscape(StringRef Values); + virtual void EmitCFIReturnColumn(int64_t Register); virtual void EmitCFIGnuArgsSize(int64_t Size); virtual void EmitCFISignalFrame(); virtual void EmitCFIUndefined(int64_t Register); virtual void EmitCFIRegister(int64_t Register1, int64_t Register2); virtual void EmitCFIWindowSave(); - virtual void EmitWinCFIStartProc(const MCSymbol *Symbol); - virtual void EmitWinCFIEndProc(); - virtual void EmitWinCFIStartChained(); - virtual void EmitWinCFIEndChained(); - virtual void EmitWinCFIPushReg(unsigned Register); - virtual void EmitWinCFISetFrame(unsigned Register, unsigned Offset); - virtual void EmitWinCFIAllocStack(unsigned Size); - virtual void EmitWinCFISaveReg(unsigned Register, unsigned Offset); - virtual void EmitWinCFISaveXMM(unsigned Register, unsigned Offset); - virtual void EmitWinCFIPushFrame(bool Code); - virtual void EmitWinCFIEndProlog(); - - virtual void EmitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except); - virtual void EmitWinEHHandlerData(); + virtual void EmitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc = SMLoc()); + virtual void EmitWinCFIEndProc(SMLoc Loc = SMLoc()); + virtual void EmitWinCFIStartChained(SMLoc Loc = SMLoc()); + virtual void EmitWinCFIEndChained(SMLoc Loc = SMLoc()); + virtual void EmitWinCFIPushReg(unsigned Register, SMLoc Loc = SMLoc()); + virtual void EmitWinCFISetFrame(unsigned Register, unsigned Offset, + SMLoc Loc = SMLoc()); + virtual void EmitWinCFIAllocStack(unsigned Size, SMLoc Loc = SMLoc()); + virtual void EmitWinCFISaveReg(unsigned Register, unsigned Offset, + SMLoc Loc = SMLoc()); + virtual void EmitWinCFISaveXMM(unsigned Register, unsigned Offset, + SMLoc Loc = SMLoc()); + virtual void EmitWinCFIPushFrame(bool Code, SMLoc Loc = SMLoc()); + virtual void EmitWinCFIEndProlog(SMLoc Loc = SMLoc()); + virtual void EmitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except, + SMLoc Loc = SMLoc()); + virtual void EmitWinEHHandlerData(SMLoc Loc = SMLoc()); /// Get the .pdata section used for the given section. Typically the given /// section is either the main .text section or some other COMDAT .text diff --git a/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h b/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h index d1d5d070bf5b..dd10881b73a8 100644 --- a/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h +++ b/contrib/llvm/include/llvm/MC/MCSubtargetInfo.h @@ -118,6 +118,10 @@ public: /// all feature bits implied by the flag. FeatureBitset ApplyFeatureFlag(StringRef FS); + /// Check whether the subtarget features are enabled/disabled as per + /// the provided string, ignoring all other features. + bool checkFeatures(StringRef FS) const; + /// getSchedModelForCPU - Get the machine model of a CPU. /// const MCSchedModel &getSchedModelForCPU(StringRef CPU) const; diff --git a/contrib/llvm/include/llvm/MC/MCSymbolWasm.h b/contrib/llvm/include/llvm/MC/MCSymbolWasm.h index 9bae6c582faa..309ebf96d1b0 100644 --- a/contrib/llvm/include/llvm/MC/MCSymbolWasm.h +++ b/contrib/llvm/include/llvm/MC/MCSymbolWasm.h @@ -18,6 +18,7 @@ class MCSymbolWasm : public MCSymbol { private: bool IsFunction = false; bool IsWeak = false; + bool IsHidden = false; std::string ModuleName; SmallVector<wasm::ValType, 1> Returns; SmallVector<wasm::ValType, 4> Params; @@ -45,6 +46,9 @@ public: bool isWeak() const { return IsWeak; } void setWeak(bool isWeak) { IsWeak = isWeak; } + bool isHidden() const { return IsHidden; } + void setHidden(bool isHidden) { IsHidden = isHidden; } + const StringRef getModuleName() const { return ModuleName; } const SmallVector<wasm::ValType, 1> &getReturns() const { diff --git a/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.def index 96179be3b8b0..5172fa44511f 100644 --- a/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/contrib/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.def @@ -19,7 +19,7 @@ #include "llvm/Support/CommandLine.h" using namespace llvm; -cl::opt<MCTargetOptions::AsmInstrumentation> AsmInstrumentation( +static cl::opt<MCTargetOptions::AsmInstrumentation> AsmInstrumentation( "asm-instrumentation", cl::desc("Instrumentation of inline assembly and " "assembly source files"), cl::init(MCTargetOptions::AsmInstrumentationNone), @@ -28,40 +28,40 @@ cl::opt<MCTargetOptions::AsmInstrumentation> AsmInstrumentation( clEnumValN(MCTargetOptions::AsmInstrumentationAddress, "address", "instrument instructions with memory arguments"))); -cl::opt<bool> RelaxAll("mc-relax-all", +static cl::opt<bool> RelaxAll("mc-relax-all", cl::desc("When used with filetype=obj, " "relax all fixups in the emitted object file")); -cl::opt<bool> IncrementalLinkerCompatible( +static cl::opt<bool> IncrementalLinkerCompatible( "incremental-linker-compatible", cl::desc( "When used with filetype=obj, " "emit an object file which can be used with an incremental linker")); -cl::opt<bool> PIECopyRelocations("pie-copy-relocations", cl::desc("PIE Copy Relocations")); +static cl::opt<bool> PIECopyRelocations("pie-copy-relocations", cl::desc("PIE Copy Relocations")); -cl::opt<int> DwarfVersion("dwarf-version", cl::desc("Dwarf version"), +static cl::opt<int> DwarfVersion("dwarf-version", cl::desc("Dwarf version"), cl::init(0)); -cl::opt<bool> ShowMCInst("asm-show-inst", +static cl::opt<bool> ShowMCInst("asm-show-inst", cl::desc("Emit internal instruction representation to " "assembly file")); -cl::opt<bool> FatalWarnings("fatal-warnings", +static cl::opt<bool> FatalWarnings("fatal-warnings", cl::desc("Treat warnings as errors")); -cl::opt<bool> NoWarn("no-warn", cl::desc("Suppress all warnings")); -cl::alias NoWarnW("W", cl::desc("Alias for --no-warn"), cl::aliasopt(NoWarn)); +static cl::opt<bool> NoWarn("no-warn", cl::desc("Suppress all warnings")); +static cl::alias NoWarnW("W", cl::desc("Alias for --no-warn"), cl::aliasopt(NoWarn)); -cl::opt<bool> NoDeprecatedWarn("no-deprecated-warn", +static cl::opt<bool> NoDeprecatedWarn("no-deprecated-warn", cl::desc("Suppress all deprecated warnings")); -cl::opt<std::string> +static cl::opt<std::string> ABIName("target-abi", cl::Hidden, cl::desc("The name of the ABI to be targeted from the backend."), cl::init("")); -static inline MCTargetOptions InitMCTargetOptionsFromFlags() { +static MCTargetOptions InitMCTargetOptionsFromFlags() { MCTargetOptions Options; Options.SanitizeAddress = (AsmInstrumentation == MCTargetOptions::AsmInstrumentationAddress); diff --git a/contrib/llvm/include/llvm/MC/MCValue.h b/contrib/llvm/include/llvm/MC/MCValue.h index aa1eaf022c55..ff223f70303b 100644 --- a/contrib/llvm/include/llvm/MC/MCValue.h +++ b/contrib/llvm/include/llvm/MC/MCValue.h @@ -38,11 +38,12 @@ class raw_ostream; /// Note that this class must remain a simple POD value class, because we need /// it to live in unions etc. class MCValue { - const MCSymbolRefExpr *SymA, *SymB; - int64_t Cst; - uint32_t RefKind; + const MCSymbolRefExpr *SymA = nullptr, *SymB = nullptr; + int64_t Cst = 0; + uint32_t RefKind = 0; + public: - MCValue() : SymA(nullptr), SymB(nullptr), Cst(0), RefKind(0) {} + MCValue() = default; int64_t getConstant() const { return Cst; } const MCSymbolRefExpr *getSymA() const { return SymA; } const MCSymbolRefExpr *getSymB() const { return SymB; } diff --git a/contrib/llvm/include/llvm/MC/MCWasmObjectWriter.h b/contrib/llvm/include/llvm/MC/MCWasmObjectWriter.h index bebc0a825810..a4d5eb857b39 100644 --- a/contrib/llvm/include/llvm/MC/MCWasmObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCWasmObjectWriter.h @@ -44,8 +44,9 @@ public: /// \param MOTW - The target specific Wasm writer subclass. /// \param OS - The stream to write to. /// \returns The constructed object writer. -MCObjectWriter *createWasmObjectWriter(MCWasmObjectTargetWriter *MOTW, - raw_pwrite_stream &OS); +std::unique_ptr<MCObjectWriter> +createWasmObjectWriter(std::unique_ptr<MCWasmObjectTargetWriter> MOTW, + raw_pwrite_stream &OS); } // End llvm namespace diff --git a/contrib/llvm/include/llvm/MC/MCWasmStreamer.h b/contrib/llvm/include/llvm/MC/MCWasmStreamer.h index bdd6f103cd44..c0d45451a9ab 100644 --- a/contrib/llvm/include/llvm/MC/MCWasmStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCWasmStreamer.h @@ -10,6 +10,8 @@ #ifndef LLVM_MC_MCWASMSTREAMER_H #define LLVM_MC_MCWASMSTREAMER_H +#include "MCAsmBackend.h" +#include "MCCodeEmitter.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCObjectStreamer.h" @@ -17,18 +19,17 @@ #include "llvm/Support/DataTypes.h" namespace llvm { -class MCAsmBackend; class MCAssembler; -class MCCodeEmitter; class MCExpr; class MCInst; class raw_ostream; class MCWasmStreamer : public MCObjectStreamer { public: - MCWasmStreamer(MCContext &Context, MCAsmBackend &TAB, raw_pwrite_stream &OS, - MCCodeEmitter *Emitter) - : MCObjectStreamer(Context, TAB, OS, Emitter), SeenIdent(false) {} + MCWasmStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, + raw_pwrite_stream &OS, std::unique_ptr<MCCodeEmitter> Emitter) + : MCObjectStreamer(Context, std::move(TAB), OS, std::move(Emitter)), + SeenIdent(false) {} ~MCWasmStreamer() override; diff --git a/contrib/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h b/contrib/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h index 198a08b5f539..3234bd93cad0 100644 --- a/contrib/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h +++ b/contrib/llvm/include/llvm/MC/MCWinCOFFObjectWriter.h @@ -10,6 +10,8 @@ #ifndef LLVM_MC_MCWINCOFFOBJECTWRITER_H #define LLVM_MC_MCWINCOFFOBJECTWRITER_H +#include <memory> + namespace llvm { class MCAsmBackend; @@ -42,8 +44,9 @@ class raw_pwrite_stream; /// \param MOTW - The target specific WinCOFF writer subclass. /// \param OS - The stream to write to. /// \returns The constructed object writer. - MCObjectWriter *createWinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, - raw_pwrite_stream &OS); + std::unique_ptr<MCObjectWriter> + createWinCOFFObjectWriter(std::unique_ptr<MCWinCOFFObjectTargetWriter> MOTW, + raw_pwrite_stream &OS); } // end namespace llvm #endif // LLVM_MC_MCWINCOFFOBJECTWRITER_H diff --git a/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h index 84e60b85be6a..a2500c06efa1 100644 --- a/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h +++ b/contrib/llvm/include/llvm/MC/MCWinCOFFStreamer.h @@ -27,8 +27,8 @@ class raw_pwrite_stream; class MCWinCOFFStreamer : public MCObjectStreamer { public: - MCWinCOFFStreamer(MCContext &Context, MCAsmBackend &MAB, MCCodeEmitter &CE, - raw_pwrite_stream &OS); + MCWinCOFFStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> MAB, + std::unique_ptr<MCCodeEmitter> CE, raw_pwrite_stream &OS); /// state management void reset() override { @@ -61,7 +61,7 @@ public: void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment) override; void EmitIdent(StringRef IdentString) override; - void EmitWinEHHandlerData() override; + void EmitWinEHHandlerData(SMLoc Loc) override; void FinishImpl() override; /// \} diff --git a/contrib/llvm/include/llvm/MC/MachineLocation.h b/contrib/llvm/include/llvm/MC/MachineLocation.h index f4fc6ee2fd19..91ed661ebeab 100644 --- a/contrib/llvm/include/llvm/MC/MachineLocation.h +++ b/contrib/llvm/include/llvm/MC/MachineLocation.h @@ -16,14 +16,14 @@ #define LLVM_MC_MACHINELOCATION_H #include <cstdint> +#include <cassert> namespace llvm { class MachineLocation { private: - bool IsRegister = false; // True if location is a register. - unsigned Register = 0; // gcc/gdb register number. - int Offset = 0; // Displacement if not register. + bool IsRegister = false; ///< True if location is a register. + unsigned Register = 0; ///< gcc/gdb register number. public: enum : uint32_t { @@ -34,13 +34,11 @@ public: MachineLocation() = default; /// Create a direct register location. - explicit MachineLocation(unsigned R) : IsRegister(true), Register(R) {} - /// Create a register-indirect location with an offset. - MachineLocation(unsigned R, int O) : Register(R), Offset(O) {} + explicit MachineLocation(unsigned R, bool Indirect = false) + : IsRegister(!Indirect), Register(R) {} bool operator==(const MachineLocation &Other) const { - return IsRegister == Other.IsRegister && Register == Other.Register && - Offset == Other.Offset; + return IsRegister == Other.IsRegister && Register == Other.Register; } // Accessors. @@ -48,24 +46,8 @@ public: bool isIndirect() const { return !IsRegister; } bool isReg() const { return IsRegister; } unsigned getReg() const { return Register; } - int getOffset() const { return Offset; } void setIsRegister(bool Is) { IsRegister = Is; } void setRegister(unsigned R) { Register = R; } - void setOffset(int O) { Offset = O; } - - /// Make this location a direct register location. - void set(unsigned R) { - IsRegister = true; - Register = R; - Offset = 0; - } - - /// Make this location a register-indirect+offset location. - void set(unsigned R, int O) { - IsRegister = false; - Register = R; - Offset = O; - } }; inline bool operator!=(const MachineLocation &LHS, const MachineLocation &RHS) { diff --git a/contrib/llvm/include/llvm/MC/SubtargetFeature.h b/contrib/llvm/include/llvm/MC/SubtargetFeature.h index cb036671b752..76c7dd560800 100644 --- a/contrib/llvm/include/llvm/MC/SubtargetFeature.h +++ b/contrib/llvm/include/llvm/MC/SubtargetFeature.h @@ -115,6 +115,9 @@ public: ArrayRef<SubtargetFeatureKV> CPUTable, ArrayRef<SubtargetFeatureKV> FeatureTable); + /// Returns the vector of individual subtarget features. + const std::vector<std::string> &getFeatures() const { return Features; } + /// Prints feature string. void print(raw_ostream &OS) const; diff --git a/contrib/llvm/include/llvm/Object/Archive.h b/contrib/llvm/include/llvm/Object/Archive.h index e56e8e464de3..5a1512bb9d36 100644 --- a/contrib/llvm/include/llvm/Object/Archive.h +++ b/contrib/llvm/include/llvm/Object/Archive.h @@ -229,7 +229,7 @@ public: enum Kind { K_GNU, - K_MIPS64, + K_GNU64, K_BSD, K_DARWIN, K_DARWIN64, diff --git a/contrib/llvm/include/llvm/Object/ArchiveWriter.h b/contrib/llvm/include/llvm/Object/ArchiveWriter.h index 1ed758d40df2..495b943d04c0 100644 --- a/contrib/llvm/include/llvm/Object/ArchiveWriter.h +++ b/contrib/llvm/include/llvm/Object/ArchiveWriter.h @@ -16,6 +16,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Object/Archive.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" namespace llvm { @@ -37,10 +38,10 @@ struct NewArchiveMember { bool Deterministic); }; -std::pair<StringRef, std::error_code> -writeArchive(StringRef ArcName, std::vector<NewArchiveMember> &NewMembers, - bool WriteSymtab, object::Archive::Kind Kind, bool Deterministic, - bool Thin, std::unique_ptr<MemoryBuffer> OldArchiveBuf = nullptr); +Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers, + bool WriteSymtab, object::Archive::Kind Kind, + bool Deterministic, bool Thin, + std::unique_ptr<MemoryBuffer> OldArchiveBuf = nullptr); } #endif diff --git a/contrib/llvm/include/llvm/Object/Binary.h b/contrib/llvm/include/llvm/Object/Binary.h index 3f5a233c1ee1..5e93691d1fd2 100644 --- a/contrib/llvm/include/llvm/Object/Binary.h +++ b/contrib/llvm/include/llvm/Object/Binary.h @@ -15,6 +15,7 @@ #define LLVM_OBJECT_BINARY_H #include "llvm/ADT/Triple.h" +#include "llvm/Object/Error.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include <algorithm> @@ -43,6 +44,8 @@ protected: ID_COFFImportFile, ID_IR, // LLVM IR + ID_WinRes, // Windows resource (.res) file. + // Object and children. ID_StartObjects, ID_COFF, @@ -57,8 +60,6 @@ protected: ID_MachO64L, // MachO 64-bit, little endian ID_MachO64B, // MachO 64-bit, big endian - ID_WinRes, // Windows resource (.res) file. - ID_Wasm, ID_EndObjects @@ -143,6 +144,16 @@ public: return Triple::ELF; return Triple::UnknownObjectFormat; } + + static std::error_code checkOffset(MemoryBufferRef M, uintptr_t Addr, + const uint64_t Size) { + if (Addr + Size < Addr || Addr + Size < Size || + Addr + Size > uintptr_t(M.getBufferEnd()) || + Addr < uintptr_t(M.getBufferStart())) { + return object_error::unexpected_eof; + } + return std::error_code(); + } }; /// @brief Create a Binary from Source, autodetecting the file type. diff --git a/contrib/llvm/include/llvm/Object/COFF.h b/contrib/llvm/include/llvm/Object/COFF.h index 89c1ba6be35f..b072dd5ba7d9 100644 --- a/contrib/llvm/include/llvm/Object/COFF.h +++ b/contrib/llvm/include/llvm/Object/COFF.h @@ -25,7 +25,6 @@ #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ErrorOr.h" #include <cassert> #include <cstddef> #include <cstdint> @@ -276,6 +275,8 @@ struct coff_symbol_generic { support::ulittle32_t Value; }; +struct coff_aux_section_definition; + class COFFSymbolRef { public: COFFSymbolRef() = default; @@ -347,6 +348,18 @@ public: return (getType() & 0xF0) >> COFF::SCT_COMPLEX_TYPE_SHIFT; } + template <typename T> const T *getAux() const { + return CS16 ? reinterpret_cast<const T *>(CS16 + 1) + : reinterpret_cast<const T *>(CS32 + 1); + } + + const coff_aux_section_definition *getSectionDefinition() const { + if (!getNumberOfAuxSymbols() || + getStorageClass() != COFF::IMAGE_SYM_CLASS_STATIC) + return nullptr; + return getAux<coff_aux_section_definition>(); + } + bool isAbsolute() const { return getSectionNumber() == -1; } @@ -730,6 +743,12 @@ struct coff_resource_dir_table { support::ulittle16_t NumberOfIDEntries; }; +struct debug_h_header { + support::ulittle32_t Magic; + support::ulittle16_t Version; + support::ulittle16_t HashAlgorithm; +}; + class COFFObjectFile : public ObjectFile { private: friend class ImportDirectoryEntryRef; @@ -753,7 +772,7 @@ private: const debug_directory *DebugDirectoryBegin; const debug_directory *DebugDirectoryEnd; // Either coff_load_configuration32 or coff_load_configuration64. - const void *LoadConfig; + const void *LoadConfig = nullptr; std::error_code getString(uint32_t offset, StringRef &Res) const; @@ -907,7 +926,7 @@ public: uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; - unsigned getArch() const override; + Triple::ArchType getArch() const override; SubtargetFeatures getFeatures() const override { return SubtargetFeatures(); } import_directory_iterator import_directory_begin() const; @@ -954,28 +973,28 @@ public: Res = reinterpret_cast<coff_symbol_type *>(getSymbolTable()) + Index; return std::error_code(); } - ErrorOr<COFFSymbolRef> getSymbol(uint32_t index) const { + Expected<COFFSymbolRef> getSymbol(uint32_t index) const { if (SymbolTable16) { const coff_symbol16 *Symb = nullptr; if (std::error_code EC = getSymbol(index, Symb)) - return EC; + return errorCodeToError(EC); return COFFSymbolRef(Symb); } if (SymbolTable32) { const coff_symbol32 *Symb = nullptr; if (std::error_code EC = getSymbol(index, Symb)) - return EC; + return errorCodeToError(EC); return COFFSymbolRef(Symb); } - return object_error::parse_failed; + return errorCodeToError(object_error::parse_failed); } template <typename T> std::error_code getAuxSymbol(uint32_t index, const T *&Res) const { - ErrorOr<COFFSymbolRef> s = getSymbol(index); - if (std::error_code EC = s.getError()) - return EC; - Res = reinterpret_cast<const T *>(s->getRawPtr()); + Expected<COFFSymbolRef> S = getSymbol(index); + if (Error E = S.takeError()) + return errorToErrorCode(std::move(E)); + Res = reinterpret_cast<const T *>(S->getRawPtr()); return std::error_code(); } @@ -1145,7 +1164,7 @@ public: BaseRelocRef() = default; BaseRelocRef(const coff_base_reloc_block_header *Header, const COFFObjectFile *Owner) - : Header(Header), Index(0), OwningObject(Owner) {} + : Header(Header), Index(0) {} bool operator==(const BaseRelocRef &Other) const; void moveNext(); @@ -1156,7 +1175,6 @@ public: private: const coff_base_reloc_block_header *Header; uint32_t Index; - const COFFObjectFile *OwningObject = nullptr; }; class ResourceSectionRef { @@ -1164,16 +1182,17 @@ public: ResourceSectionRef() = default; explicit ResourceSectionRef(StringRef Ref) : BBS(Ref, support::little) {} - ErrorOr<ArrayRef<UTF16>> getEntryNameString(const coff_resource_dir_entry &Entry); - ErrorOr<const coff_resource_dir_table &> + Expected<ArrayRef<UTF16>> + getEntryNameString(const coff_resource_dir_entry &Entry); + Expected<const coff_resource_dir_table &> getEntrySubDir(const coff_resource_dir_entry &Entry); - ErrorOr<const coff_resource_dir_table &> getBaseTable(); + Expected<const coff_resource_dir_table &> getBaseTable(); private: BinaryByteStream BBS; - ErrorOr<const coff_resource_dir_table &> getTableAtOffset(uint32_t Offset); - ErrorOr<ArrayRef<UTF16>> getDirStringAtOffset(uint32_t Offset); + Expected<const coff_resource_dir_table &> getTableAtOffset(uint32_t Offset); + Expected<ArrayRef<UTF16>> getDirStringAtOffset(uint32_t Offset); }; // Corresponds to `_FPO_DATA` structure in the PE/COFF spec. diff --git a/contrib/llvm/include/llvm/Object/COFFImportFile.h b/contrib/llvm/include/llvm/Object/COFFImportFile.h index cf9c80a06f49..4b284de679b3 100644 --- a/contrib/llvm/include/llvm/Object/COFFImportFile.h +++ b/contrib/llvm/include/llvm/Object/COFFImportFile.h @@ -96,11 +96,9 @@ struct COFFShortExport { } }; -std::error_code writeImportLibrary(StringRef ImportName, - StringRef Path, - ArrayRef<COFFShortExport> Exports, - COFF::MachineTypes Machine, - bool MakeWeakAliases); +Error writeImportLibrary(StringRef ImportName, StringRef Path, + ArrayRef<COFFShortExport> Exports, + COFF::MachineTypes Machine, bool MakeWeakAliases); } // namespace object } // namespace llvm diff --git a/contrib/llvm/include/llvm/Object/ELF.h b/contrib/llvm/include/llvm/Object/ELF.h index 670c0bbce3ac..45c98233dec0 100644 --- a/contrib/llvm/include/llvm/Object/ELF.h +++ b/contrib/llvm/include/llvm/Object/ELF.h @@ -83,6 +83,8 @@ public: private: StringRef Buf; + ELFFile(StringRef Object); + public: const Elf_Ehdr *getHeader() const { return reinterpret_cast<const Elf_Ehdr *>(base()); @@ -102,8 +104,6 @@ public: Expected<ArrayRef<Elf_Word>> getSHNDXTable(const Elf_Shdr &Section, Elf_Shdr_Range Sections) const; - void VerifyStrTab(const Elf_Shdr *sh) const; - StringRef getRelocationTypeName(uint32_t Type) const; void getRelocationTypeName(uint32_t Type, SmallVectorImpl<char> &Result) const; @@ -112,7 +112,7 @@ public: Expected<const Elf_Sym *> getRelocationSymbol(const Elf_Rel *Rel, const Elf_Shdr *SymTab) const; - ELFFile(StringRef Object); + static Expected<ELFFile> create(StringRef Object); bool isMipsELF64() const { return getHeader()->e_machine == ELF::EM_MIPS && @@ -140,10 +140,16 @@ public: return getSectionContentsAsArray<Elf_Rel>(Sec); } + Expected<std::vector<Elf_Rela>> android_relas(const Elf_Shdr *Sec) const; + /// \brief Iterate over program header table. Expected<Elf_Phdr_Range> program_headers() const { if (getHeader()->e_phnum && getHeader()->e_phentsize != sizeof(Elf_Phdr)) return createError("invalid e_phentsize"); + if (getHeader()->e_phoff + + (getHeader()->e_phnum * getHeader()->e_phentsize) > + getBufSize()) + return createError("program headers longer than binary"); auto *Begin = reinterpret_cast<const Elf_Phdr *>(base() + getHeader()->e_phoff); return makeArrayRef(Begin, Begin + getHeader()->e_phnum); @@ -271,6 +277,9 @@ ELFFile<ELFT>::getSectionContentsAsArray(const Elf_Shdr *Sec) const { Offset + Size > Buf.size()) return createError("invalid section offset"); + if (Offset % alignof(T)) + return createError("unaligned data"); + const T *Start = reinterpret_cast<const T *>(base() + Offset); return makeArrayRef(Start, Size / sizeof(T)); } @@ -341,14 +350,13 @@ ELFFile<ELFT>::getSectionStringTable(Elf_Shdr_Range Sections) const { return getStringTable(&Sections[Index]); } -template <class ELFT> -ELFFile<ELFT>::ELFFile(StringRef Object) : Buf(Object) { - assert(sizeof(Elf_Ehdr) <= Buf.size() && "Invalid buffer"); -} +template <class ELFT> ELFFile<ELFT>::ELFFile(StringRef Object) : Buf(Object) {} template <class ELFT> -bool compareAddr(uint64_t VAddr, const Elf_Phdr_Impl<ELFT> *Phdr) { - return VAddr < Phdr->p_vaddr; +Expected<ELFFile<ELFT>> ELFFile<ELFT>::create(StringRef Object) { + if (sizeof(Elf_Ehdr) > Object.size()) + return createError("Invalid buffer"); + return ELFFile(Object); } template <class ELFT> diff --git a/contrib/llvm/include/llvm/Object/ELFObjectFile.h b/contrib/llvm/include/llvm/Object/ELFObjectFile.h index 73011f6f9fe1..40503cb6bb9d 100644 --- a/contrib/llvm/include/llvm/Object/ELFObjectFile.h +++ b/contrib/llvm/include/llvm/Object/ELFObjectFile.h @@ -33,7 +33,6 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" #include <cassert> #include <cstdint> @@ -61,7 +60,7 @@ protected: virtual uint64_t getSectionFlags(DataRefImpl Sec) const = 0; virtual uint64_t getSectionOffset(DataRefImpl Sec) const = 0; - virtual ErrorOr<int64_t> getRelocationAddend(DataRefImpl Rel) const = 0; + virtual Expected<int64_t> getRelocationAddend(DataRefImpl Rel) const = 0; public: using elf_symbol_iterator_range = iterator_range<elf_symbol_iterator>; @@ -167,7 +166,7 @@ public: return cast<ELFObjectFileBase>(RelocationRef::getObject()); } - ErrorOr<int64_t> getAddend() const { + Expected<int64_t> getAddend() const { return getObject()->getRelocationAddend(getRawDataRefImpl()); } }; @@ -210,6 +209,11 @@ public: using Elf_Rela = typename ELFFile<ELFT>::Elf_Rela; using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn; +private: + ELFObjectFile(MemoryBufferRef Object, ELFFile<ELFT> EF, + const Elf_Shdr *DotDynSymSec, const Elf_Shdr *DotSymtabSec, + ArrayRef<Elf_Word> ShndxTable); + protected: ELFFile<ELFT> EF; @@ -328,7 +332,8 @@ protected: bool isDyldELFObject; public: - ELFObjectFile(MemoryBufferRef Object, std::error_code &EC); + ELFObjectFile(ELFObjectFile<ELFT> &&Other); + static Expected<ELFObjectFile<ELFT>> create(MemoryBufferRef Object); const Elf_Rel *getRel(DataRefImpl Rel) const; const Elf_Rela *getRela(DataRefImpl Rela) const; @@ -353,11 +358,11 @@ public: section_iterator section_begin() const override; section_iterator section_end() const override; - ErrorOr<int64_t> getRelocationAddend(DataRefImpl Rel) const override; + Expected<int64_t> getRelocationAddend(DataRefImpl Rel) const override; uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; - unsigned getArch() const override; + Triple::ArchType getArch() const override; std::error_code getPlatformFlags(unsigned &Result) const override { Result = EF.getHeader()->e_flags; @@ -667,6 +672,10 @@ std::error_code ELFObjectFile<ELFT>::getSectionContents(DataRefImpl Sec, StringRef &Result) const { const Elf_Shdr *EShdr = getSection(Sec); + if (std::error_code EC = + checkOffset(getMemoryBufferRef(), + (uintptr_t)base() + EShdr->sh_offset, EShdr->sh_size)) + return EC; Result = StringRef((const char *)base() + EShdr->sh_offset, EShdr->sh_size); return std::error_code(); } @@ -812,10 +821,10 @@ void ELFObjectFile<ELFT>::getRelocationTypeName( } template <class ELFT> -ErrorOr<int64_t> +Expected<int64_t> ELFObjectFile<ELFT>::getRelocationAddend(DataRefImpl Rel) const { if (getRelSection(Rel)->sh_type != ELF::SHT_RELA) - return object_error::parse_failed; + return createError("Section is not SHT_RELA"); return (int64_t)getRela(Rel)->r_addend; } @@ -840,50 +849,64 @@ ELFObjectFile<ELFT>::getRela(DataRefImpl Rela) const { } template <class ELFT> -ELFObjectFile<ELFT>::ELFObjectFile(MemoryBufferRef Object, std::error_code &EC) - : ELFObjectFileBase( - getELFType(ELFT::TargetEndianness == support::little, ELFT::Is64Bits), - Object), - EF(Data.getBuffer()) { +Expected<ELFObjectFile<ELFT>> +ELFObjectFile<ELFT>::create(MemoryBufferRef Object) { + auto EFOrErr = ELFFile<ELFT>::create(Object.getBuffer()); + if (Error E = EFOrErr.takeError()) + return std::move(E); + auto EF = std::move(*EFOrErr); + auto SectionsOrErr = EF.sections(); - if (!SectionsOrErr) { - EC = errorToErrorCode(SectionsOrErr.takeError()); - return; - } + if (!SectionsOrErr) + return SectionsOrErr.takeError(); + + const Elf_Shdr *DotDynSymSec = nullptr; + const Elf_Shdr *DotSymtabSec = nullptr; + ArrayRef<Elf_Word> ShndxTable; for (const Elf_Shdr &Sec : *SectionsOrErr) { switch (Sec.sh_type) { case ELF::SHT_DYNSYM: { - if (DotDynSymSec) { - // More than one .dynsym! - EC = object_error::parse_failed; - return; - } + if (DotDynSymSec) + return createError("More than one dynamic symbol table!"); DotDynSymSec = &Sec; break; } case ELF::SHT_SYMTAB: { - if (DotSymtabSec) { - // More than one .dynsym! - EC = object_error::parse_failed; - return; - } + if (DotSymtabSec) + return createError("More than one static symbol table!"); DotSymtabSec = &Sec; break; } case ELF::SHT_SYMTAB_SHNDX: { auto TableOrErr = EF.getSHNDXTable(Sec); - if (!TableOrErr) { - EC = errorToErrorCode(TableOrErr.takeError()); - return; - } + if (!TableOrErr) + return TableOrErr.takeError(); ShndxTable = *TableOrErr; break; } } } + return ELFObjectFile<ELFT>(Object, EF, DotDynSymSec, DotSymtabSec, + ShndxTable); } template <class ELFT> +ELFObjectFile<ELFT>::ELFObjectFile(MemoryBufferRef Object, ELFFile<ELFT> EF, + const Elf_Shdr *DotDynSymSec, + const Elf_Shdr *DotSymtabSec, + ArrayRef<Elf_Word> ShndxTable) + : ELFObjectFileBase( + getELFType(ELFT::TargetEndianness == support::little, ELFT::Is64Bits), + Object), + EF(EF), DotDynSymSec(DotDynSymSec), DotSymtabSec(DotSymtabSec), + ShndxTable(ShndxTable) {} + +template <class ELFT> +ELFObjectFile<ELFT>::ELFObjectFile(ELFObjectFile<ELFT> &&Other) + : ELFObjectFile(Other.Data, Other.EF, Other.DotDynSymSec, + Other.DotSymtabSec, Other.ShndxTable) {} + +template <class ELFT> basic_symbol_iterator ELFObjectFile<ELFT>::symbol_begin() const { DataRefImpl Sym = toDRI(DotSymtabSec, 0); return basic_symbol_iterator(SymbolRef(Sym, this)); @@ -991,9 +1014,7 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { case ELF::EM_WEBASSEMBLY: return "ELF64-wasm"; case ELF::EM_AMDGPU: - return (EF.getHeader()->e_ident[ELF::EI_OSABI] == ELF::ELFOSABI_AMDGPU_HSA - && IsLittleEndian) ? - "ELF64-amdgpu-hsacobj" : "ELF64-amdgpu"; + return "ELF64-amdgpu"; case ELF::EM_BPF: return "ELF64-BPF"; default: @@ -1005,8 +1026,7 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { } } -template <class ELFT> -unsigned ELFObjectFile<ELFT>::getArch() const { +template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const { bool IsLittleEndian = ELFT::TargetEndianness == support::little; switch (EF.getHeader()->e_machine) { case ELF::EM_386: @@ -1061,11 +1081,20 @@ unsigned ELFObjectFile<ELFT>::getArch() const { default: return Triple::UnknownArch; } - case ELF::EM_AMDGPU: - return (EF.getHeader()->e_ident[ELF::EI_CLASS] == ELF::ELFCLASS64 - && EF.getHeader()->e_ident[ELF::EI_OSABI] == ELF::ELFOSABI_AMDGPU_HSA - && IsLittleEndian) ? - Triple::amdgcn : Triple::UnknownArch; + case ELF::EM_AMDGPU: { + if (!IsLittleEndian) + return Triple::UnknownArch; + + unsigned EFlags = EF.getHeader()->e_flags; + switch (EFlags & ELF::EF_AMDGPU_ARCH) { + case ELF::EF_AMDGPU_ARCH_R600: + return Triple::r600; + case ELF::EF_AMDGPU_ARCH_GCN: + return Triple::amdgcn; + default: + return Triple::UnknownArch; + } + } case ELF::EM_BPF: return IsLittleEndian ? Triple::bpfel : Triple::bpfeb; diff --git a/contrib/llvm/include/llvm/Object/ELFTypes.h b/contrib/llvm/include/llvm/Object/ELFTypes.h index 808144694acb..83b688548fdc 100644 --- a/contrib/llvm/include/llvm/Object/ELFTypes.h +++ b/contrib/llvm/include/llvm/Object/ELFTypes.h @@ -406,10 +406,10 @@ struct Elf_Rel_Impl<ELFType<TargetEndianness, false>, false> { return (unsigned char)(this->getRInfo(isMips64EL) & 0x0ff); } void setSymbol(uint32_t s, bool IsMips64EL) { - setSymbolAndType(s, getType(), IsMips64EL); + setSymbolAndType(s, getType(IsMips64EL), IsMips64EL); } void setType(unsigned char t, bool IsMips64EL) { - setSymbolAndType(getSymbol(), t, IsMips64EL); + setSymbolAndType(getSymbol(IsMips64EL), t, IsMips64EL); } void setSymbolAndType(uint32_t s, unsigned char t, bool IsMips64EL) { this->setRInfo((s << 8) + t, IsMips64EL); @@ -459,10 +459,10 @@ struct Elf_Rel_Impl<ELFType<TargetEndianness, true>, false> { return (uint32_t)(this->getRInfo(isMips64EL) & 0xffffffffL); } void setSymbol(uint32_t s, bool IsMips64EL) { - setSymbolAndType(s, getType(), IsMips64EL); + setSymbolAndType(s, getType(IsMips64EL), IsMips64EL); } void setType(uint32_t t, bool IsMips64EL) { - setSymbolAndType(getSymbol(), t, IsMips64EL); + setSymbolAndType(getSymbol(IsMips64EL), t, IsMips64EL); } void setSymbolAndType(uint32_t s, uint32_t t, bool IsMips64EL) { this->setRInfo(((uint64_t)s << 32) + (t & 0xffffffffL), IsMips64EL); diff --git a/contrib/llvm/include/llvm/Object/IRObjectFile.h b/contrib/llvm/include/llvm/Object/IRObjectFile.h index 9a696bffd1f0..6c271b1a1f44 100644 --- a/contrib/llvm/include/llvm/Object/IRObjectFile.h +++ b/contrib/llvm/include/llvm/Object/IRObjectFile.h @@ -52,12 +52,12 @@ public: /// \brief Finds and returns bitcode embedded in the given object file, or an /// error code if not found. - static ErrorOr<MemoryBufferRef> findBitcodeInObject(const ObjectFile &Obj); + static Expected<MemoryBufferRef> findBitcodeInObject(const ObjectFile &Obj); /// \brief Finds and returns bitcode in the given memory buffer (which may /// be either a bitcode file or a native object file with embedded bitcode), /// or an error code if not found. - static ErrorOr<MemoryBufferRef> + static Expected<MemoryBufferRef> findBitcodeInMemBuffer(MemoryBufferRef Object); static Expected<std::unique_ptr<IRObjectFile>> create(MemoryBufferRef Object, diff --git a/contrib/llvm/include/llvm/Object/IRSymtab.h b/contrib/llvm/include/llvm/Object/IRSymtab.h index 824a67a672fa..5f6a024cd132 100644 --- a/contrib/llvm/include/llvm/Object/IRSymtab.h +++ b/contrib/llvm/include/llvm/Object/IRSymtab.h @@ -121,6 +121,9 @@ struct Uncommon { /// COFF-specific: the name of the symbol that a weak external resolves to /// if not defined. Str COFFWeakExternFallbackName; + + /// Specified section name, if any. + Str SectionName; }; struct Header { @@ -128,7 +131,7 @@ struct Header { /// when the format changes, but it does not need to be incremented if a /// change to LLVM would cause it to create a different symbol table. Word Version; - enum { kCurrentVersion = 0 }; + enum { kCurrentVersion = 1 }; /// The producer's version string (LLVM_VERSION_STRING " " LLVM_REVISION). /// Consumers should rebuild the symbol table from IR if the producer's @@ -165,6 +168,7 @@ struct Symbol { // Copied from storage::Uncommon. uint32_t CommonSize, CommonAlign; StringRef COFFWeakExternFallbackName; + StringRef SectionName; /// Returns the mangled symbol name. StringRef getName() const { return Name; } @@ -215,6 +219,8 @@ struct Symbol { assert(isWeak() && isIndirect()); return COFFWeakExternFallbackName; } + + StringRef getSectionName() const { return SectionName; } }; /// This class can be used to read a Symtab and Strtab produced by @@ -300,7 +306,10 @@ class Reader::SymbolRef : public Symbol { CommonSize = UncI->CommonSize; CommonAlign = UncI->CommonAlign; COFFWeakExternFallbackName = R->str(UncI->COFFWeakExternFallbackName); - } + SectionName = R->str(UncI->SectionName); + } else + // Reset this field so it can be queried unconditionally for all symbols. + SectionName = ""; } public: diff --git a/contrib/llvm/include/llvm/Object/MachO.h b/contrib/llvm/include/llvm/Object/MachO.h index 2c3c89d10546..d0cc40da4293 100644 --- a/contrib/llvm/include/llvm/Object/MachO.h +++ b/contrib/llvm/include/llvm/Object/MachO.h @@ -66,11 +66,13 @@ using dice_iterator = content_iterator<DiceRef>; /// ExportEntry encapsulates the current-state-of-the-walk used when doing a /// non-recursive walk of the trie data structure. This allows you to iterate /// across all exported symbols using: -/// for (const llvm::object::ExportEntry &AnExport : Obj->exports()) { +/// Error Err; +/// for (const llvm::object::ExportEntry &AnExport : Obj->exports(&Err)) { /// } +/// if (Err) { report error ... class ExportEntry { public: - ExportEntry(ArrayRef<uint8_t> Trie); + ExportEntry(Error *Err, const MachOObjectFile *O, ArrayRef<uint8_t> Trie); StringRef name() const; uint64_t flags() const; @@ -88,7 +90,7 @@ private: void moveToFirst(); void moveToEnd(); - uint64_t readULEB128(const uint8_t *&p); + uint64_t readULEB128(const uint8_t *&p, const char **error); void pushDownUntilBottom(); void pushNode(uint64_t Offset); @@ -107,12 +109,19 @@ private: unsigned ParentStringLength = 0; bool IsExportNode = false; }; + using NodeList = SmallVector<NodeState, 16>; + using node_iterator = NodeList::const_iterator; + Error *E; + const MachOObjectFile *O; ArrayRef<uint8_t> Trie; SmallString<256> CumulativeString; - SmallVector<NodeState, 16> Stack; - bool Malformed = false; + NodeList Stack; bool Done = false; + + iterator_range<node_iterator> nodes() const { + return make_range(Stack.begin(), Stack.end()); + } }; using export_iterator = content_iterator<ExportEntry>; @@ -301,6 +310,16 @@ public: bool isSectionBSS(DataRefImpl Sec) const override; bool isSectionVirtual(DataRefImpl Sec) const override; bool isSectionBitcode(DataRefImpl Sec) const override; + + /// When dsymutil generates the companion file, it strips all unnecessary + /// sections (e.g. everything in the _TEXT segment) by omitting their body + /// and setting the offset in their corresponding load command to zero. + /// + /// While the load command itself is valid, reading the section corresponds + /// to reading the number of bytes specified in the load command, starting + /// from offset 0 (i.e. the Mach-O header at the beginning of the file). + bool isSectionStripped(DataRefImpl Sec) const override; + relocation_iterator section_rel_begin(DataRefImpl Sec) const override; relocation_iterator section_rel_end(DataRefImpl Sec) const override; @@ -310,6 +329,9 @@ public: return make_range(extrel_begin(), extrel_end()); } + relocation_iterator locrel_begin() const; + relocation_iterator locrel_end() const; + void moveRelocationNext(DataRefImpl &Rel) const override; uint64_t getRelocationOffset(DataRefImpl Rel) const override; symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override; @@ -341,7 +363,7 @@ public: uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; - unsigned getArch() const override; + Triple::ArchType getArch() const override; SubtargetFeatures getFeatures() const override { return SubtargetFeatures(); } Triple getArchTriple(const char **McpuDefault = nullptr) const; @@ -356,10 +378,13 @@ public: iterator_range<load_command_iterator> load_commands() const; /// For use iterating over all exported symbols. - iterator_range<export_iterator> exports() const; + iterator_range<export_iterator> exports(Error &Err) const; /// For use examining a trie not in a MachOObjectFile. - static iterator_range<export_iterator> exports(ArrayRef<uint8_t> Trie); + static iterator_range<export_iterator> exports(Error &Err, + ArrayRef<uint8_t> Trie, + const MachOObjectFile *O = + nullptr); /// For use iterating over all rebase table entries. iterator_range<rebase_iterator> rebaseTable(Error &Err); diff --git a/contrib/llvm/include/llvm/Object/ObjectFile.h b/contrib/llvm/include/llvm/Object/ObjectFile.h index afcad3090703..079a59468156 100644 --- a/contrib/llvm/include/llvm/Object/ObjectFile.h +++ b/contrib/llvm/include/llvm/Object/ObjectFile.h @@ -15,6 +15,7 @@ #define LLVM_OBJECT_OBJECTFILE_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/MC/SubtargetFeature.h" @@ -23,7 +24,6 @@ #include "llvm/Object/SymbolicFile.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include <cassert> @@ -109,6 +109,7 @@ public: bool isBSS() const; bool isVirtual() const; bool isBitcode() const; + bool isStripped() const; bool containsSymbol(SymbolRef S) const; @@ -236,6 +237,7 @@ protected: // A section is 'virtual' if its contents aren't present in the object image. virtual bool isSectionVirtual(DataRefImpl Sec) const = 0; virtual bool isSectionBitcode(DataRefImpl Sec) const; + virtual bool isSectionStripped(DataRefImpl Sec) const; virtual relocation_iterator section_rel_begin(DataRefImpl Sec) const = 0; virtual relocation_iterator section_rel_end(DataRefImpl Sec) const = 0; virtual section_iterator getRelocatedSection(DataRefImpl Sec) const; @@ -278,10 +280,13 @@ public: virtual uint8_t getBytesInAddress() const = 0; virtual StringRef getFileFormatName() const = 0; - virtual /* Triple::ArchType */ unsigned getArch() const = 0; + virtual Triple::ArchType getArch() const = 0; virtual SubtargetFeatures getFeatures() const = 0; virtual void setARMSubArch(Triple &TheTriple) const { } + /// @brief Create a triple from the data in this object file. + Triple makeTriple() const; + /// Returns platform-specific object flags, if any. virtual std::error_code getPlatformFlags(unsigned &Result) const { Result = 0; @@ -317,10 +322,10 @@ public: return v->isObject(); } - static ErrorOr<std::unique_ptr<COFFObjectFile>> + static Expected<std::unique_ptr<COFFObjectFile>> createCOFFObjectFile(MemoryBufferRef Object); - static ErrorOr<std::unique_ptr<ObjectFile>> + static Expected<std::unique_ptr<ObjectFile>> createELFObjectFile(MemoryBufferRef Object); static Expected<std::unique_ptr<MachOObjectFile>> @@ -439,6 +444,10 @@ inline bool SectionRef::isBitcode() const { return OwningObject->isSectionBitcode(SectionPimpl); } +inline bool SectionRef::isStripped() const { + return OwningObject->isSectionStripped(SectionPimpl); +} + inline relocation_iterator SectionRef::relocation_begin() const { return OwningObject->section_rel_begin(SectionPimpl); } diff --git a/contrib/llvm/include/llvm/Object/RelocVisitor.h b/contrib/llvm/include/llvm/Object/RelocVisitor.h index c358d3996435..2d0e938f06fd 100644 --- a/contrib/llvm/include/llvm/Object/RelocVisitor.h +++ b/contrib/llvm/include/llvm/Object/RelocVisitor.h @@ -25,7 +25,6 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ErrorOr.h" #include <cstdint> #include <system_error> @@ -115,9 +114,10 @@ private: } int64_t getELFAddend(RelocationRef R) { - ErrorOr<int64_t> AddendOrErr = ELFRelocationRef(R).getAddend(); - if (std::error_code EC = AddendOrErr.getError()) - report_fatal_error(EC.message()); + Expected<int64_t> AddendOrErr = ELFRelocationRef(R).getAddend(); + handleAllErrors(AddendOrErr.takeError(), [](const ErrorInfoBase &EI) { + report_fatal_error(EI.message()); + }); return *AddendOrErr; } @@ -169,6 +169,8 @@ private: return (Value + getELFAddend(R)) & 0xFFFFFFFF; case ELF::R_MIPS_64: return Value + getELFAddend(R); + case ELF::R_MIPS_TLS_DTPREL64: + return Value + getELFAddend(R) - 0x8000; } HasError = true; return 0; @@ -260,8 +262,11 @@ private: } uint64_t visitMips32(uint32_t Rel, RelocationRef R, uint64_t Value) { + // FIXME: Take in account implicit addends to get correct results. if (Rel == ELF::R_MIPS_32) return Value & 0xFFFFFFFF; + if (Rel == ELF::R_MIPS_TLS_DTPREL32) + return Value & 0xFFFFFFFF; HasError = true; return 0; } @@ -297,6 +302,8 @@ private: return Value; } break; + default: + break; } HasError = true; return 0; diff --git a/contrib/llvm/include/llvm/Object/StackMapParser.h b/contrib/llvm/include/llvm/Object/StackMapParser.h index 0c5e1e38cbaa..557db5afa825 100644 --- a/contrib/llvm/include/llvm/Object/StackMapParser.h +++ b/contrib/llvm/include/llvm/Object/StackMapParser.h @@ -62,7 +62,7 @@ public: uint64_t getStackSize() const { return read<uint64_t>(P + sizeof(uint64_t)); } - + /// Get the number of callsite records. uint64_t getRecordCount() const { return read<uint64_t>(P + (2 * sizeof(uint64_t))); diff --git a/contrib/llvm/include/llvm/Object/Wasm.h b/contrib/llvm/include/llvm/Object/Wasm.h index 07ee4a4d6c4d..5bb1a3fca3d1 100644 --- a/contrib/llvm/include/llvm/Object/Wasm.h +++ b/contrib/llvm/include/llvm/Object/Wasm.h @@ -43,25 +43,56 @@ public: }; WasmSymbol(StringRef Name, SymbolType Type, uint32_t Section, - uint32_t ElementIndex) - : Name(Name), Type(Type), Section(Section), ElementIndex(ElementIndex) {} + uint32_t ElementIndex, uint32_t ImportIndex = 0) + : Name(Name), Type(Type), Section(Section), ElementIndex(ElementIndex), + ImportIndex(ImportIndex) {} StringRef Name; SymbolType Type; uint32_t Section; uint32_t Flags = 0; - // Index into the imports, exports or functions array of the object depending - // on the type + // Index into either the function or global index space. uint32_t ElementIndex; + // For imports, the index into the import table + uint32_t ImportIndex; + + bool isFunction() const { + return Type == WasmSymbol::SymbolType::FUNCTION_IMPORT || + Type == WasmSymbol::SymbolType::FUNCTION_EXPORT || + Type == WasmSymbol::SymbolType::DEBUG_FUNCTION_NAME; + } + + bool isWeak() const { - return Flags & wasm::WASM_SYMBOL_FLAG_WEAK; + return getBinding() == wasm::WASM_SYMBOL_BINDING_WEAK; + } + + bool isGlobal() const { + return getBinding() == wasm::WASM_SYMBOL_BINDING_GLOBAL; + } + + bool isLocal() const { + return getBinding() == wasm::WASM_SYMBOL_BINDING_LOCAL; + } + + unsigned getBinding() const { + return Flags & wasm::WASM_SYMBOL_BINDING_MASK; + } + + bool isHidden() const { + return getVisibility() == wasm::WASM_SYMBOL_VISIBILITY_HIDDEN; + } + + unsigned getVisibility() const { + return Flags & wasm::WASM_SYMBOL_VISIBILITY_MASK; } void print(raw_ostream &Out) const { Out << "Name=" << Name << ", Type=" << static_cast<int>(Type) - << ", Flags=" << Flags << " ElemIndex=" << ElementIndex; + << ", Flags=" << Flags << " ElemIndex=" << ElementIndex + << ", ImportIndex=" << ImportIndex; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -97,29 +128,18 @@ public: static bool classof(const Binary *v) { return v->isWasm(); } - const std::vector<wasm::WasmSignature>& types() const { return Signatures; } - const std::vector<uint32_t>& functionTypes() const { return FunctionTypes; } - const std::vector<wasm::WasmImport>& imports() const { return Imports; } - const std::vector<wasm::WasmTable>& tables() const { return Tables; } - const std::vector<wasm::WasmLimits>& memories() const { return Memories; } - const std::vector<wasm::WasmGlobal>& globals() const { return Globals; } - const std::vector<wasm::WasmExport>& exports() const { return Exports; } + ArrayRef<wasm::WasmSignature> types() const { return Signatures; } + ArrayRef<uint32_t> functionTypes() const { return FunctionTypes; } + ArrayRef<wasm::WasmImport> imports() const { return Imports; } + ArrayRef<wasm::WasmTable> tables() const { return Tables; } + ArrayRef<wasm::WasmLimits> memories() const { return Memories; } + ArrayRef<wasm::WasmGlobal> globals() const { return Globals; } + ArrayRef<wasm::WasmExport> exports() const { return Exports; } const wasm::WasmLinkingData& linkingData() const { return LinkingData; } - - uint32_t getNumberOfSymbols() const { - return Symbols.size(); - } - - const std::vector<wasm::WasmElemSegment>& elements() const { - return ElemSegments; - } - - const std::vector<WasmSegment>& dataSegments() const { - return DataSegments; - } - - const std::vector<wasm::WasmFunction>& functions() const { return Functions; } - const ArrayRef<uint8_t>& code() const { return CodeSection; } + uint32_t getNumberOfSymbols() const { return Symbols.size(); } + ArrayRef<wasm::WasmElemSegment> elements() const { return ElemSegments; } + ArrayRef<WasmSegment> dataSegments() const { return DataSegments; } + ArrayRef<wasm::WasmFunction> functions() const { return Functions; } uint32_t startFunction() const { return StartFunction; } void moveSymbolNext(DataRefImpl &Symb) const override; @@ -132,6 +152,7 @@ public: Expected<StringRef> getSymbolName(DataRefImpl Symb) const override; Expected<uint64_t> getSymbolAddress(DataRefImpl Symb) const override; + uint64_t getWasmSymbolValue(const WasmSymbol& Sym) const; uint64_t getSymbolValueImpl(DataRefImpl Symb) const override; uint32_t getSymbolAlignment(DataRefImpl Symb) const override; uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override; @@ -169,11 +190,12 @@ public: section_iterator section_end() const override; uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; - unsigned getArch() const override; + Triple::ArchType getArch() const override; SubtargetFeatures getFeatures() const override; bool isRelocatableObject() const override; private: + bool isValidFunctionIndex(uint32_t Index) const; const WasmSection &getWasmSection(DataRefImpl Ref) const; const wasm::WasmRelocation &getWasmRelocation(DataRefImpl Ref) const; @@ -204,6 +226,8 @@ private: Error parseRelocSection(StringRef Name, const uint8_t *Ptr, const uint8_t *End); + void populateSymbolTable(); + wasm::WasmObjectHeader Header; std::vector<WasmSection> Sections; std::vector<wasm::WasmSignature> Signatures; @@ -217,10 +241,13 @@ private: std::vector<WasmSegment> DataSegments; std::vector<wasm::WasmFunction> Functions; std::vector<WasmSymbol> Symbols; - ArrayRef<uint8_t> CodeSection; uint32_t StartFunction = -1; bool HasLinkingSection = false; wasm::WasmLinkingData LinkingData; + uint32_t NumImportedGlobals = 0; + uint32_t NumImportedFunctions = 0; + uint32_t ImportSection = 0; + uint32_t ExportSection = 0; StringMap<uint32_t> SymbolMap; }; diff --git a/contrib/llvm/include/llvm/Object/WindowsResource.h b/contrib/llvm/include/llvm/Object/WindowsResource.h index 3d32409fd4ac..a077c82871bf 100644 --- a/contrib/llvm/include/llvm/Object/WindowsResource.h +++ b/contrib/llvm/include/llvm/Object/WindowsResource.h @@ -85,6 +85,12 @@ struct WinResHeaderSuffix { support::ulittle32_t Characteristics; }; +class EmptyResError : public GenericBinaryError { +public: + EmptyResError(Twine Msg, object_error ECOverride) + : GenericBinaryError(Msg, ECOverride) {} +}; + class ResourceEntryRef { public: Error moveNext(bool &End); @@ -94,7 +100,9 @@ public: bool checkNameString() const { return IsStringName; } ArrayRef<UTF16> getNameString() const { return Name; } uint16_t getNameID() const { return NameID; } + uint16_t getDataVersion() const { return Suffix->DataVersion; } uint16_t getLanguage() const { return Suffix->Language; } + uint16_t getMemoryFlags() const { return Suffix->MemoryFlags; } uint16_t getMajorVersion() const { return Suffix->Version >> 16; } uint16_t getMinorVersion() const { return Suffix->Version; } uint32_t getCharacteristics() const { return Suffix->Characteristics; } @@ -103,11 +111,12 @@ public: private: friend class WindowsResource; - ResourceEntryRef(BinaryStreamRef Ref, const WindowsResource *Owner, - Error &Err); - + ResourceEntryRef(BinaryStreamRef Ref, const WindowsResource *Owner); Error loadNext(); + static Expected<ResourceEntryRef> create(BinaryStreamRef Ref, + const WindowsResource *Owner); + BinaryStreamReader Reader; bool IsStringType; ArrayRef<UTF16> Type; @@ -117,7 +126,6 @@ private: uint16_t NameID; const WinResHeaderSuffix *Suffix = nullptr; ArrayRef<uint8_t> Data; - const WindowsResource *OwningRes = nullptr; }; class WindowsResource : public Binary { diff --git a/contrib/llvm/include/llvm/ObjectYAML/COFFYAML.h b/contrib/llvm/include/llvm/ObjectYAML/COFFYAML.h index bbceefac3d94..8794eaa6d59a 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/COFFYAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/COFFYAML.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/ObjectYAML/CodeViewYAMLDebugSections.h" +#include "llvm/ObjectYAML/CodeViewYAMLTypeHashing.h" #include "llvm/ObjectYAML/CodeViewYAMLTypes.h" #include "llvm/ObjectYAML/YAML.h" #include <cstdint> @@ -66,6 +67,7 @@ struct Section { yaml::BinaryRef SectionData; std::vector<CodeViewYAML::YAMLDebugSubsection> DebugS; std::vector<CodeViewYAML::LeafRecord> DebugT; + Optional<CodeViewYAML::DebugHSection> DebugH; std::vector<Relocation> Relocations; StringRef Name; @@ -158,6 +160,16 @@ struct ScalarEnumerationTraits<COFF::RelocationTypeAMD64> { }; template <> +struct ScalarEnumerationTraits<COFF::RelocationTypesARM> { + static void enumeration(IO &IO, COFF::RelocationTypesARM &Value); +}; + +template <> +struct ScalarEnumerationTraits<COFF::RelocationTypesARM64> { + static void enumeration(IO &IO, COFF::RelocationTypesARM64 &Value); +}; + +template <> struct ScalarEnumerationTraits<COFF::WindowsSubsystem> { static void enumeration(IO &IO, COFF::WindowsSubsystem &Value); }; diff --git a/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h b/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h new file mode 100644 index 000000000000..4f0d9efb963b --- /dev/null +++ b/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h @@ -0,0 +1,62 @@ +//==- CodeViewYAMLTypeHashing.h - CodeView YAMLIO Type hashing ----*- C++-*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines classes for handling the YAML representation of CodeView +// Debug Info. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECTYAML_CODEVIEWYAMLTYPEHASHING_H +#define LLVM_OBJECTYAML_CODEVIEWYAMLTYPEHASHING_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/YAMLTraits.h" +#include <cstdint> +#include <memory> +#include <vector> + +namespace llvm { + +namespace CodeViewYAML { + +struct GlobalHash { + GlobalHash() = default; + explicit GlobalHash(StringRef S) : Hash(S) { + assert(S.size() == 20 && "Invalid hash size!"); + } + explicit GlobalHash(ArrayRef<uint8_t> S) : Hash(S) { + assert(S.size() == 20 && "Invalid hash size!"); + } + yaml::BinaryRef Hash; +}; + +struct DebugHSection { + uint32_t Magic; + uint16_t Version; + uint16_t HashAlgorithm; + std::vector<GlobalHash> Hashes; +}; + +DebugHSection fromDebugH(ArrayRef<uint8_t> DebugT); +ArrayRef<uint8_t> toDebugH(const DebugHSection &DebugH, + BumpPtrAllocator &Alloc); + +} // end namespace CodeViewYAML + +} // end namespace llvm + +LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::DebugHSection) +LLVM_YAML_DECLARE_SCALAR_TRAITS(CodeViewYAML::GlobalHash, QuotingType::None) +LLVM_YAML_IS_SEQUENCE_VECTOR(CodeViewYAML::GlobalHash) + +#endif // LLVM_OBJECTYAML_CODEVIEWYAMLTYPES_H diff --git a/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypes.h b/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypes.h index 88a5668f0a14..bc3b5567c2f9 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypes.h +++ b/contrib/llvm/include/llvm/ObjectYAML/CodeViewYAMLTypes.h @@ -27,10 +27,8 @@ namespace llvm { namespace codeview { - -class TypeTableBuilder; - -} // end namespace codeview +class AppendingTypeTableBuilder; +} namespace CodeViewYAML { @@ -48,8 +46,8 @@ struct MemberRecord { struct LeafRecord { std::shared_ptr<detail::LeafRecordBase> Leaf; - codeview::CVType toCodeViewRecord(BumpPtrAllocator &Allocator) const; - codeview::CVType toCodeViewRecord(codeview::TypeTableBuilder &TS) const; + codeview::CVType + toCodeViewRecord(codeview::AppendingTypeTableBuilder &Serializer) const; static Expected<LeafRecord> fromCodeViewRecord(codeview::CVType Type); }; @@ -60,7 +58,7 @@ ArrayRef<uint8_t> toDebugT(ArrayRef<LeafRecord>, BumpPtrAllocator &Alloc); } // end namespace llvm -LLVM_YAML_DECLARE_SCALAR_TRAITS(codeview::GUID, true) +LLVM_YAML_DECLARE_SCALAR_TRAITS(codeview::GUID, QuotingType::Single) LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::LeafRecord) LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::MemberRecord) diff --git a/contrib/llvm/include/llvm/ObjectYAML/ELFYAML.h b/contrib/llvm/include/llvm/ObjectYAML/ELFYAML.h index ed455311696e..7ba83967330e 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/ELFYAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/ELFYAML.h @@ -37,17 +37,20 @@ namespace ELFYAML { // In the future, these would probably be better suited by C++11 enum // class's with appropriate fixed underlying type. LLVM_YAML_STRONG_TYPEDEF(uint16_t, ELF_ET) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_PT) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_EM) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFCLASS) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFDATA) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFOSABI) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_EF) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_PF) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_SHT) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_REL) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_RSS) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_SHF) +LLVM_YAML_STRONG_TYPEDEF(uint16_t, ELF_SHN) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STT) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STV) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STO) @@ -71,10 +74,24 @@ struct FileHeader { llvm::yaml::Hex64 Entry; }; +struct SectionName { + StringRef Section; +}; + +struct ProgramHeader { + ELF_PT Type; + ELF_PF Flags; + llvm::yaml::Hex64 VAddr; + llvm::yaml::Hex64 PAddr; + Optional<llvm::yaml::Hex64> Align; + std::vector<SectionName> Sections; +}; + struct Symbol { StringRef Name; ELF_STT Type; StringRef Section; + Optional<ELF_SHN> Index; llvm::yaml::Hex64 Value; llvm::yaml::Hex64 Size; uint8_t Other; @@ -147,7 +164,7 @@ struct Relocation { llvm::yaml::Hex64 Offset; int64_t Addend; ELF_REL Type; - StringRef Symbol; + Optional<StringRef> Symbol; }; struct RelocationSection : Section { @@ -183,21 +200,25 @@ struct MipsABIFlags : Section { struct Object { FileHeader Header; + std::vector<ProgramHeader> ProgramHeaders; std::vector<std::unique_ptr<Section>> Sections; // Although in reality the symbols reside in a section, it is a lot // cleaner and nicer if we read them from the YAML as a separate // top-level key, which automatically ensures that invariants like there // being a single SHT_SYMTAB section are upheld. LocalGlobalWeakSymbols Symbols; + LocalGlobalWeakSymbols DynamicSymbols; }; } // end namespace ELFYAML } // end namespace llvm +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader) LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Section>) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Symbol) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Relocation) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::SectionOrType) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::SectionName) namespace llvm { namespace yaml { @@ -207,6 +228,10 @@ struct ScalarEnumerationTraits<ELFYAML::ELF_ET> { static void enumeration(IO &IO, ELFYAML::ELF_ET &Value); }; +template <> struct ScalarEnumerationTraits<ELFYAML::ELF_PT> { + static void enumeration(IO &IO, ELFYAML::ELF_PT &Value); +}; + template <> struct ScalarEnumerationTraits<ELFYAML::ELF_EM> { static void enumeration(IO &IO, ELFYAML::ELF_EM &Value); @@ -232,6 +257,10 @@ struct ScalarBitSetTraits<ELFYAML::ELF_EF> { static void bitset(IO &IO, ELFYAML::ELF_EF &Value); }; +template <> struct ScalarBitSetTraits<ELFYAML::ELF_PF> { + static void bitset(IO &IO, ELFYAML::ELF_PF &Value); +}; + template <> struct ScalarEnumerationTraits<ELFYAML::ELF_SHT> { static void enumeration(IO &IO, ELFYAML::ELF_SHT &Value); @@ -242,6 +271,10 @@ struct ScalarBitSetTraits<ELFYAML::ELF_SHF> { static void bitset(IO &IO, ELFYAML::ELF_SHF &Value); }; +template <> struct ScalarEnumerationTraits<ELFYAML::ELF_SHN> { + static void enumeration(IO &IO, ELFYAML::ELF_SHN &Value); +}; + template <> struct ScalarEnumerationTraits<ELFYAML::ELF_STT> { static void enumeration(IO &IO, ELFYAML::ELF_STT &Value); @@ -302,9 +335,14 @@ struct MappingTraits<ELFYAML::FileHeader> { static void mapping(IO &IO, ELFYAML::FileHeader &FileHdr); }; +template <> struct MappingTraits<ELFYAML::ProgramHeader> { + static void mapping(IO &IO, ELFYAML::ProgramHeader &FileHdr); +}; + template <> struct MappingTraits<ELFYAML::Symbol> { static void mapping(IO &IO, ELFYAML::Symbol &Symbol); + static StringRef validate(IO &IO, ELFYAML::Symbol &Symbol); }; template <> @@ -331,6 +369,10 @@ template <> struct MappingTraits<ELFYAML::SectionOrType> { static void mapping(IO &IO, ELFYAML::SectionOrType §ionOrType); }; +template <> struct MappingTraits<ELFYAML::SectionName> { + static void mapping(IO &IO, ELFYAML::SectionName §ionName); +}; + } // end namespace yaml } // end namespace llvm diff --git a/contrib/llvm/include/llvm/ObjectYAML/MachOYAML.h b/contrib/llvm/include/llvm/ObjectYAML/MachOYAML.h index 305497b6aa6a..1fa8f92e516a 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/MachOYAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/MachOYAML.h @@ -261,17 +261,17 @@ using char_16 = char[16]; template <> struct ScalarTraits<char_16> { static void output(const char_16 &Val, void *, raw_ostream &Out); static StringRef input(StringRef Scalar, void *, char_16 &Val); - static bool mustQuote(StringRef S); + static QuotingType mustQuote(StringRef S); }; // This trait is used for UUIDs. It reads and writes them matching otool's // formatting style. -using uuid_t = uint8_t[16]; +using uuid_t = raw_ostream::uuid_t; template <> struct ScalarTraits<uuid_t> { static void output(const uuid_t &Val, void *, raw_ostream &Out); static StringRef input(StringRef Scalar, void *, uuid_t &Val); - static bool mustQuote(StringRef S); + static QuotingType mustQuote(StringRef S); }; // Load Command struct mapping traits diff --git a/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h b/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h index 709ad8ec3b77..188ce8e44491 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/WasmYAML.h @@ -34,13 +34,16 @@ LLVM_YAML_STRONG_TYPEDEF(int32_t, SignatureForm) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ExportKind) LLVM_YAML_STRONG_TYPEDEF(uint32_t, Opcode) LLVM_YAML_STRONG_TYPEDEF(uint32_t, RelocType) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, SymbolFlags) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, SegmentFlags) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, LimitFlags) struct FileHeader { yaml::Hex32 Version; }; struct Limits { - yaml::Hex32 Flags; + LimitFlags Flags; yaml::Hex32 Initial; yaml::Hex32 Maximum; }; @@ -109,6 +112,13 @@ struct NameEntry { StringRef Name; }; +struct SegmentInfo { + uint32_t Index; + StringRef Name; + uint32_t Alignment; + SegmentFlags Flags; +}; + struct Signature { uint32_t Index; SignatureForm Form = wasm::WASM_TYPE_FUNC; @@ -118,7 +128,12 @@ struct Signature { struct SymbolInfo { StringRef Name; - uint32_t Flags; + SymbolFlags Flags; +}; + +struct InitFunction { + uint32_t Priority; + uint32_t FunctionIndex; }; struct Section { @@ -160,9 +175,10 @@ struct LinkingSection : CustomSection { return C && C->Name == "linking"; } - std::vector<SymbolInfo> SymbolInfos; uint32_t DataSize; - uint32_t DataAlignment; + std::vector<SymbolInfo> SymbolInfos; + std::vector<SegmentInfo> SegmentInfos; + std::vector<InitFunction> InitFunctions; }; struct TypeSection : Section { @@ -297,7 +313,9 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Function) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::LocalDecl) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Relocation) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::NameEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SegmentInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SymbolInfo) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::InitFunction) namespace llvm { namespace yaml { @@ -326,6 +344,18 @@ template <> struct MappingTraits<WasmYAML::Global> { static void mapping(IO &IO, WasmYAML::Global &Global); }; +template <> struct ScalarBitSetTraits<WasmYAML::LimitFlags> { + static void bitset(IO &IO, WasmYAML::LimitFlags &Value); +}; + +template <> struct ScalarBitSetTraits<WasmYAML::SymbolFlags> { + static void bitset(IO &IO, WasmYAML::SymbolFlags &Value); +}; + +template <> struct ScalarBitSetTraits<WasmYAML::SegmentFlags> { + static void bitset(IO &IO, WasmYAML::SegmentFlags &Value); +}; + template <> struct ScalarEnumerationTraits<WasmYAML::SectionType> { static void enumeration(IO &IO, WasmYAML::SectionType &Type); }; @@ -354,6 +384,10 @@ template <> struct MappingTraits<WasmYAML::NameEntry> { static void mapping(IO &IO, WasmYAML::NameEntry &NameEntry); }; +template <> struct MappingTraits<WasmYAML::SegmentInfo> { + static void mapping(IO &IO, WasmYAML::SegmentInfo &SegmentInfo); +}; + template <> struct MappingTraits<WasmYAML::LocalDecl> { static void mapping(IO &IO, WasmYAML::LocalDecl &LocalDecl); }; @@ -374,6 +408,10 @@ template <> struct MappingTraits<WasmYAML::SymbolInfo> { static void mapping(IO &IO, WasmYAML::SymbolInfo &Info); }; +template <> struct MappingTraits<WasmYAML::InitFunction> { + static void mapping(IO &IO, WasmYAML::InitFunction &Init); +}; + template <> struct ScalarEnumerationTraits<WasmYAML::ValueType> { static void enumeration(IO &IO, WasmYAML::ValueType &Type); }; diff --git a/contrib/llvm/include/llvm/ObjectYAML/YAML.h b/contrib/llvm/include/llvm/ObjectYAML/YAML.h index 29151a269df0..93266dd67f1a 100644 --- a/contrib/llvm/include/llvm/ObjectYAML/YAML.h +++ b/contrib/llvm/include/llvm/ObjectYAML/YAML.h @@ -107,7 +107,7 @@ inline bool operator==(const BinaryRef &LHS, const BinaryRef &RHS) { template <> struct ScalarTraits<BinaryRef> { static void output(const BinaryRef &, void *, raw_ostream &); static StringRef input(StringRef, void *, BinaryRef &); - static bool mustQuote(StringRef S) { return needsQuotes(S); } + static QuotingType mustQuote(StringRef S) { return needsQuotes(S); } }; } // end namespace yaml diff --git a/contrib/llvm/include/llvm/Option/OptParser.td b/contrib/llvm/include/llvm/Option/OptParser.td index 481223698719..9c373741770b 100644 --- a/contrib/llvm/include/llvm/Option/OptParser.td +++ b/contrib/llvm/include/llvm/Option/OptParser.td @@ -93,6 +93,7 @@ class Option<list<string> prefixes, string name, OptionKind kind> { string HelpText = ?; string MetaVarName = ?; string Values = ?; + code ValuesCode = ?; list<OptionFlag> Flags = []; OptionGroup Group = ?; Option Alias = ?; @@ -128,6 +129,7 @@ class Group<OptionGroup group> { OptionGroup Group = group; } class HelpText<string text> { string HelpText = text; } class MetaVarName<string name> { string MetaVarName = name; } class Values<string value> { string Values = value; } +class ValuesCode<code valuecode> { code ValuesCode = valuecode; } // Predefined options. diff --git a/contrib/llvm/include/llvm/Option/OptTable.h b/contrib/llvm/include/llvm/Option/OptTable.h index a35e182f00e5..57a6954f4878 100644 --- a/contrib/llvm/include/llvm/Option/OptTable.h +++ b/contrib/llvm/include/llvm/Option/OptTable.h @@ -57,8 +57,8 @@ public: }; private: - /// \brief The static option information table. - ArrayRef<Info> OptionInfos; + /// \brief The option information table. + std::vector<Info> OptionInfos; bool IgnoreCase; unsigned TheInputOptionID = 0; @@ -143,6 +143,17 @@ public: std::vector<std::string> findByPrefix(StringRef Cur, unsigned short DisableFlags) const; + /// Add Values to Option's Values class + /// + /// \param [in] Option - Prefix + Name of the flag which Values will be + /// changed. For example, "-analyzer-checker". + /// \param [in] Values - String of Values seperated by ",", such as + /// "foo, bar..", where foo and bar is the argument which the Option flag + /// takes + /// + /// \return true in success, and false in fail. + bool addValues(const char *Option, const char *Values); + /// \brief Parse a single argument; returning the new argument and /// updating Index. /// @@ -191,12 +202,16 @@ public: /// \param FlagsToInclude - If non-zero, only include options with any /// of these flags set. /// \param FlagsToExclude - Exclude options with any of these flags set. - void PrintHelp(raw_ostream &OS, const char *Name, - const char *Title, unsigned FlagsToInclude, - unsigned FlagsToExclude) const; - - void PrintHelp(raw_ostream &OS, const char *Name, - const char *Title, bool ShowHidden = false) const; + /// \param ShowAllAliases - If true, display all options including aliases + /// that don't have help texts. By default, we display + /// only options that are not hidden and have help + /// texts. + void PrintHelp(raw_ostream &OS, const char *Name, const char *Title, + unsigned FlagsToInclude, unsigned FlagsToExclude, + bool ShowAllAliases) const; + + void PrintHelp(raw_ostream &OS, const char *Name, const char *Title, + bool ShowHidden = false, bool ShowAllAliases = false) const; }; } // end namespace opt diff --git a/contrib/llvm/include/llvm/Pass.h b/contrib/llvm/include/llvm/Pass.h index 2dd6935cf01c..a29b3771abb4 100644 --- a/contrib/llvm/include/llvm/Pass.h +++ b/contrib/llvm/include/llvm/Pass.h @@ -29,24 +29,24 @@ #ifndef LLVM_PASS_H #define LLVM_PASS_H +#include "llvm/ADT/StringRef.h" #include <string> namespace llvm { +class AnalysisResolver; +class AnalysisUsage; class BasicBlock; class Function; +class ImmutablePass; class Module; -class AnalysisUsage; class PassInfo; -class ImmutablePass; -class PMStack; -class AnalysisResolver; class PMDataManager; +class PMStack; class raw_ostream; -class StringRef; // AnalysisID - Use the PassInfo to identify a pass... -typedef const void* AnalysisID; +using AnalysisID = const void *; /// Different types of internal pass managers. External pass managers /// (PassManager and FunctionPassManager) are not represented here. @@ -79,24 +79,21 @@ enum PassKind { /// constrained passes described below. /// class Pass { - AnalysisResolver *Resolver; // Used to resolve analysis + AnalysisResolver *Resolver = nullptr; // Used to resolve analysis const void *PassID; PassKind Kind; - void operator=(const Pass&) = delete; - Pass(const Pass &) = delete; public: - explicit Pass(PassKind K, char &pid) - : Resolver(nullptr), PassID(&pid), Kind(K) { } + explicit Pass(PassKind K, char &pid) : PassID(&pid), Kind(K) {} + Pass(const Pass &) = delete; + Pass &operator=(const Pass &) = delete; virtual ~Pass(); - PassKind getPassKind() const { return Kind; } /// getPassName - Return a nice clean name for a pass. This usually /// implemented in terms of the name that is registered by one of the /// Registration templates, but can be overloaded directly. - /// virtual StringRef getPassName() const; /// getPassID - Return the PassID number that corresponds to this pass. @@ -106,12 +103,10 @@ public: /// doInitialization - Virtual method overridden by subclasses to do /// any necessary initialization before any pass is run. - /// virtual bool doInitialization(Module &) { return false; } /// doFinalization - Virtual method overriden by subclasses to do any /// necessary clean up after all passes have run. - /// virtual bool doFinalization(Module &) { return false; } /// print - Print out the internal state of the pass. This is called by @@ -120,19 +115,20 @@ public: /// null. This automatically forwards to a virtual function that does not /// provide the Module* in case the analysis doesn't need it it can just be /// ignored. - /// - virtual void print(raw_ostream &O, const Module *M) const; + virtual void print(raw_ostream &OS, const Module *M) const; + void dump() const; // dump - Print to stderr. /// createPrinterPass - Get a Pass appropriate to print the IR this /// pass operates on (Module, Function or MachineFunction). - virtual Pass *createPrinterPass(raw_ostream &O, + virtual Pass *createPrinterPass(raw_ostream &OS, const std::string &Banner) const = 0; /// Each pass is responsible for assigning a pass manager to itself. /// PMS is the stack of available pass manager. virtual void assignPassManager(PMStack &, PassManagerType) {} + /// Check if available pass managers are suitable for this pass or not. virtual void preparePassManager(PMStack &); @@ -147,7 +143,6 @@ public: /// analysis information to do their job. If a pass specifies that it uses a /// particular analysis result to this function, it can then use the /// getAnalysis<AnalysisType>() function, below. - /// virtual void getAnalysisUsage(AnalysisUsage &) const; /// releaseMemory() - This member can be implemented by a pass if it wants to @@ -160,7 +155,6 @@ public: /// /// Optionally implement this function to release pass memory when it is no /// longer used. - /// virtual void releaseMemory(); /// getAdjustedAnalysisPointer - This method is used when a pass implements @@ -197,7 +191,6 @@ public: /// the case when the analysis is not available. This method is often used by /// transformation APIs to update analysis results for a pass automatically as /// the transform is performed. - /// template<typename AnalysisType> AnalysisType * getAnalysisIfAvailable() const; // Defined in PassAnalysisSupport.h @@ -206,13 +199,11 @@ public: /// obviously cannot give you a properly typed instance of the class if you /// don't have the class name available (use getAnalysisIfAvailable if you /// do), but it can tell you if you need to preserve the pass at least. - /// bool mustPreserveAnalysisID(char &AID) const; /// getAnalysis<AnalysisType>() - This function is used by subclasses to get /// to the analysis information that they claim to use by overriding the /// getAnalysisUsage function. - /// template<typename AnalysisType> AnalysisType &getAnalysis() const; // Defined in PassAnalysisSupport.h @@ -226,7 +217,6 @@ public: AnalysisType &getAnalysisID(AnalysisID PI, Function &F); }; - //===----------------------------------------------------------------------===// /// ModulePass class - This class is used to implement unstructured /// interprocedural optimizations and analyses. ModulePasses may do anything @@ -234,8 +224,13 @@ public: /// class ModulePass : public Pass { public: + explicit ModulePass(char &pid) : Pass(PT_Module, pid) {} + + // Force out-of-line virtual method. + ~ModulePass() override; + /// createPrinterPass - Get a module printer pass. - Pass *createPrinterPass(raw_ostream &O, + Pass *createPrinterPass(raw_ostream &OS, const std::string &Banner) const override; /// runOnModule - Virtual method overriden by subclasses to process the module @@ -247,17 +242,12 @@ public: /// Return what kind of Pass Manager can manage this pass. PassManagerType getPotentialPassManagerType() const override; - explicit ModulePass(char &pid) : Pass(PT_Module, pid) {} - // Force out-of-line virtual method. - ~ModulePass() override; - protected: /// Optional passes call this function to check whether the pass should be /// skipped. This is the case when optimization bisect is over the limit. bool skipModule(Module &M) const; }; - //===----------------------------------------------------------------------===// /// ImmutablePass class - This class is used to provide information that does /// not need to be run. This is useful for things like target information and @@ -265,25 +255,22 @@ protected: /// class ImmutablePass : public ModulePass { public: + explicit ImmutablePass(char &pid) : ModulePass(pid) {} + + // Force out-of-line virtual method. + ~ImmutablePass() override; + /// initializePass - This method may be overriden by immutable passes to allow /// them to perform various initialization actions they require. This is /// primarily because an ImmutablePass can "require" another ImmutablePass, /// and if it does, the overloaded version of initializePass may get access to /// these passes with getAnalysis<>. - /// virtual void initializePass(); ImmutablePass *getAsImmutablePass() override { return this; } /// ImmutablePasses are never run. - /// bool runOnModule(Module &) override { return false; } - - explicit ImmutablePass(char &pid) - : ModulePass(pid) {} - - // Force out-of-line virtual method. - ~ImmutablePass() override; }; //===----------------------------------------------------------------------===// @@ -300,12 +287,11 @@ public: explicit FunctionPass(char &pid) : Pass(PT_Function, pid) {} /// createPrinterPass - Get a function printer pass. - Pass *createPrinterPass(raw_ostream &O, + Pass *createPrinterPass(raw_ostream &OS, const std::string &Banner) const override; /// runOnFunction - Virtual method overriden by subclasses to do the /// per-function processing of the pass. - /// virtual bool runOnFunction(Function &F) = 0; void assignPassManager(PMStack &PMS, PassManagerType T) override; @@ -320,8 +306,6 @@ protected: bool skipFunction(const Function &F) const; }; - - //===----------------------------------------------------------------------===// /// BasicBlockPass class - This class is used to implement most local /// optimizations. Optimizations should subclass this class if they @@ -337,7 +321,7 @@ public: explicit BasicBlockPass(char &pid) : Pass(PT_BasicBlock, pid) {} /// createPrinterPass - Get a basic block printer pass. - Pass *createPrinterPass(raw_ostream &O, + Pass *createPrinterPass(raw_ostream &OS, const std::string &Banner) const override; using llvm::Pass::doInitialization; @@ -345,17 +329,14 @@ public: /// doInitialization - Virtual method overridden by BasicBlockPass subclasses /// to do any necessary per-function initialization. - /// virtual bool doInitialization(Function &); /// runOnBasicBlock - Virtual method overriden by subclasses to do the /// per-basicblock processing of the pass. - /// virtual bool runOnBasicBlock(BasicBlock &BB) = 0; /// doFinalization - Virtual method overriden by BasicBlockPass subclasses to /// do any post processing needed after all passes have run. - /// virtual bool doFinalization(Function &); void assignPassManager(PMStack &PMS, PassManagerType T) override; @@ -379,12 +360,19 @@ extern bool TimePassesIsEnabled; // debugging options like -print-after-all/-print-before-all. // @brief Tells if the function IR should be printed by PrinterPass. extern bool isFunctionInPrintList(StringRef FunctionName); -} // End llvm namespace + +/// forcePrintModuleIR - returns true if IR printing passes should +// be printing module IR (even for local-pass printers e.g. function-pass) +// to provide more context, as enabled by debugging option -print-module-scope +// @brief Tells if IR printer should be printing module IR +extern bool forcePrintModuleIR(); + +} // end namespace llvm // Include support files that contain important APIs commonly used by Passes, // but that we want to separate out to make it easier to read the header files. -// +#include "llvm/InitializePasses.h" #include "llvm/PassAnalysisSupport.h" #include "llvm/PassSupport.h" -#endif +#endif // LLVM_PASS_H diff --git a/contrib/llvm/include/llvm/PassAnalysisSupport.h b/contrib/llvm/include/llvm/PassAnalysisSupport.h index abd992938057..b109605355bf 100644 --- a/contrib/llvm/include/llvm/PassAnalysisSupport.h +++ b/contrib/llvm/include/llvm/PassAnalysisSupport.h @@ -19,12 +19,18 @@ #ifndef LLVM_PASSANALYSISSUPPORT_H #define LLVM_PASSANALYSISSUPPORT_H +#include "Pass.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Pass.h" +#include "llvm/ADT/StringRef.h" +#include <cassert> +#include <utility> #include <vector> namespace llvm { -class StringRef; + +class Function; +class Pass; +class PMDataManager; //===----------------------------------------------------------------------===// /// Represent the analysis usage information of a pass. This tracks analyses @@ -36,7 +42,7 @@ class StringRef; /// class AnalysisUsage { public: - typedef SmallVectorImpl<AnalysisID> VectorType; + using VectorType = SmallVectorImpl<AnalysisID>; private: /// Sets of analyses required and preserved by a pass @@ -47,10 +53,10 @@ private: SmallVector<AnalysisID, 2> RequiredTransitive; SmallVector<AnalysisID, 2> Preserved; SmallVector<AnalysisID, 0> Used; - bool PreservesAll; + bool PreservesAll = false; public: - AnalysisUsage() : PreservesAll(false) {} + AnalysisUsage() = default; ///@{ /// Add the specified ID to the required set of the usage info for a pass. @@ -124,7 +130,6 @@ public: /// /// This function annotates the AnalysisUsage info object to say that analyses /// that only depend on the CFG are preserved by this pass. - /// void setPreservesCFG(); const VectorType &getRequiredSet() const { return Required; } @@ -140,15 +145,12 @@ public: /// analysis information out of pass manager that is responsible to manage /// the pass. /// -class PMDataManager; class AnalysisResolver { -private: - AnalysisResolver() = delete; - public: - explicit AnalysisResolver(PMDataManager &P) : PM(P) { } + AnalysisResolver() = delete; + explicit AnalysisResolver(PMDataManager &P) : PM(P) {} - inline PMDataManager &getPMDataManager() { return PM; } + PMDataManager &getPMDataManager() { return PM; } /// Find pass that is implementing PI. Pass *findImplPass(AnalysisID PI) { @@ -183,7 +185,7 @@ public: private: /// This keeps track of which passes implements the interfaces that are /// required by the current pass (to implement getAnalysis()). - std::vector<std::pair<AnalysisID, Pass*> > AnalysisImpls; + std::vector<std::pair<AnalysisID, Pass *>> AnalysisImpls; /// PassManager that is used to resolve analysis info PMDataManager &PM; @@ -196,7 +198,6 @@ private: /// the case when the analysis is not available. This method is often used by /// transformation APIs to update analysis results for a pass automatically as /// the transform is performed. -/// template<typename AnalysisType> AnalysisType *Pass::getAnalysisIfAvailable() const { assert(Resolver && "Pass not resident in a PassManager object!"); @@ -216,7 +217,6 @@ AnalysisType *Pass::getAnalysisIfAvailable() const { /// getAnalysis<AnalysisType>() - This function is used by subclasses to get /// to the analysis information that they claim to use by overriding the /// getAnalysisUsage function. -/// template<typename AnalysisType> AnalysisType &Pass::getAnalysis() const { assert(Resolver && "Pass has not been inserted into a PassManager object!"); @@ -231,9 +231,9 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI) const { // should be a small number, we just do a linear search over a (dense) // vector. Pass *ResultPass = Resolver->findImplPass(PI); - assert (ResultPass && - "getAnalysis*() called on an analysis that was not " - "'required' by pass!"); + assert(ResultPass && + "getAnalysis*() called on an analysis that was not " + "'required' by pass!"); // Because the AnalysisType may not be a subclass of pass (for // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially @@ -245,7 +245,6 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI) const { /// getAnalysis<AnalysisType>() - This function is used by subclasses to get /// to the analysis information that they claim to use by overriding the /// getAnalysisUsage function. -/// template<typename AnalysisType> AnalysisType &Pass::getAnalysis(Function &F) { assert(Resolver &&"Pass has not been inserted into a PassManager object!"); @@ -270,6 +269,6 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI, Function &F) { return *(AnalysisType*)ResultPass->getAdjustedAnalysisPointer(PI); } -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_PASSANALYSISSUPPORT_H diff --git a/contrib/llvm/include/llvm/PassInfo.h b/contrib/llvm/include/llvm/PassInfo.h index 81dface3c9a0..2f1ab4d43377 100644 --- a/contrib/llvm/include/llvm/PassInfo.h +++ b/contrib/llvm/include/llvm/PassInfo.h @@ -10,18 +10,17 @@ // This file defines and implements the PassInfo class. // //===----------------------------------------------------------------------===// + #ifndef LLVM_PASSINFO_H #define LLVM_PASSINFO_H #include "llvm/ADT/StringRef.h" - #include <cassert> #include <vector> namespace llvm { class Pass; -class TargetMachine; //===--------------------------------------------------------------------------- /// PassInfo class - An instance of this class exists for every pass known by @@ -31,18 +30,17 @@ class TargetMachine; /// class PassInfo { public: - typedef Pass* (*NormalCtor_t)(); + using NormalCtor_t = Pass* (*)(); private: StringRef PassName; // Nice name for Pass StringRef PassArgument; // Command Line argument to run this pass const void *PassID; - const bool IsCFGOnlyPass; // Pass only looks at the CFG. + const bool IsCFGOnlyPass = false; // Pass only looks at the CFG. const bool IsAnalysis; // True if an analysis pass. const bool IsAnalysisGroup; // True if an analysis group. std::vector<const PassInfo *> ItfImpl; // Interfaces implemented by this pass - - NormalCtor_t NormalCtor; + NormalCtor_t NormalCtor = nullptr; public: /// PassInfo ctor - Do not call this directly, this should only be invoked @@ -51,21 +49,22 @@ public: bool isCFGOnly, bool is_analysis) : PassName(name), PassArgument(arg), PassID(pi), IsCFGOnlyPass(isCFGOnly), IsAnalysis(is_analysis), IsAnalysisGroup(false), NormalCtor(normal) {} + /// PassInfo ctor - Do not call this directly, this should only be invoked /// through RegisterPass. This version is for use by analysis groups; it /// does not auto-register the pass. PassInfo(StringRef name, const void *pi) - : PassName(name), PassArgument(""), PassID(pi), IsCFGOnlyPass(false), - IsAnalysis(false), IsAnalysisGroup(true), NormalCtor(nullptr) {} + : PassName(name), PassID(pi), IsAnalysis(false), IsAnalysisGroup(true) {} + + PassInfo(const PassInfo &) = delete; + PassInfo &operator=(const PassInfo &) = delete; /// getPassName - Return the friendly name for the pass, never returns null - /// StringRef getPassName() const { return PassName; } /// getPassArgument - Return the command line option that may be passed to /// 'opt' that will cause this pass to be run. This will return null if there /// is no argument. - /// StringRef getPassArgument() const { return PassArgument; } /// getTypeInfo - Return the id object for the pass... @@ -77,7 +76,6 @@ public: /// isAnalysisGroup - Return true if this is an analysis group, not a normal /// pass. - /// bool isAnalysisGroup() const { return IsAnalysisGroup; } bool isAnalysis() const { return IsAnalysis; } @@ -88,7 +86,6 @@ public: /// getNormalCtor - Return a pointer to a function, that when called, creates /// an instance of the pass and returns it. This pointer may be null if there /// is no default constructor for the pass. - /// NormalCtor_t getNormalCtor() const { return NormalCtor; } @@ -108,23 +105,17 @@ public: /// addInterfaceImplemented - This method is called when this pass is /// registered as a member of an analysis group with the RegisterAnalysisGroup /// template. - /// void addInterfaceImplemented(const PassInfo *ItfPI) { ItfImpl.push_back(ItfPI); } /// getInterfacesImplemented - Return a list of all of the analysis group /// interfaces implemented by this pass. - /// const std::vector<const PassInfo*> &getInterfacesImplemented() const { return ItfImpl; } - -private: - void operator=(const PassInfo &) = delete; - PassInfo(const PassInfo &) = delete; }; -} +} // end namespace llvm -#endif +#endif // LLVM_PASSINFO_H diff --git a/contrib/llvm/include/llvm/PassRegistry.h b/contrib/llvm/include/llvm/PassRegistry.h index 4bb19675585e..93edc12bdc7b 100644 --- a/contrib/llvm/include/llvm/PassRegistry.h +++ b/contrib/llvm/include/llvm/PassRegistry.h @@ -18,16 +18,15 @@ #define LLVM_PASSREGISTRY_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" -#include "llvm/PassInfo.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/RWMutex.h" +#include <memory> #include <vector> namespace llvm { -class StringRef; class PassInfo; struct PassRegistrationListener; @@ -41,17 +40,17 @@ class PassRegistry { mutable sys::SmartRWMutex<true> Lock; /// PassInfoMap - Keep track of the PassInfo object for each registered pass. - typedef DenseMap<const void *, const PassInfo *> MapType; + using MapType = DenseMap<const void *, const PassInfo *>; MapType PassInfoMap; - typedef StringMap<const PassInfo *> StringMapType; + using StringMapType = StringMap<const PassInfo *>; StringMapType PassInfoStringMap; std::vector<std::unique_ptr<const PassInfo>> ToFree; std::vector<PassRegistrationListener *> Listeners; public: - PassRegistry() {} + PassRegistry() = default; ~PassRegistry(); /// getPassRegistry - Access the global registry object, which is @@ -94,6 +93,6 @@ public: // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassRegistry, LLVMPassRegistryRef) -} +} // end namespace llvm -#endif +#endif // LLVM_PASSREGISTRY_H diff --git a/contrib/llvm/include/llvm/PassSupport.h b/contrib/llvm/include/llvm/PassSupport.h index 602f45ac5178..1bf23dcba50b 100644 --- a/contrib/llvm/include/llvm/PassSupport.h +++ b/contrib/llvm/include/llvm/PassSupport.h @@ -21,16 +21,16 @@ #ifndef LLVM_PASSSUPPORT_H #define LLVM_PASSSUPPORT_H -#include "Pass.h" -#include "llvm/InitializePasses.h" +#include "llvm/ADT/StringRef.h" #include "llvm/PassInfo.h" #include "llvm/PassRegistry.h" -#include "llvm/Support/Atomic.h" #include "llvm/Support/Threading.h" #include <functional> namespace llvm { +class Pass; + #define INITIALIZE_PASS(passName, arg, name, cfg, analysis) \ static void *initialize##passName##PassOnce(PassRegistry &Registry) { \ PassInfo *PI = new PassInfo( \ @@ -88,7 +88,6 @@ template <typename PassName> Pass *callDefaultCtor() { return new PassName(); } /// /// This statement will cause your pass to be created by calling the default /// constructor exposed by the pass. -/// template <typename passName> struct RegisterPass : public PassInfo { // Register Pass using default constructor... RegisterPass(StringRef PassArg, StringRef Name, bool CFGOnly = false, @@ -118,7 +117,6 @@ template <typename passName> struct RegisterPass : public PassInfo { /// The actual interface may also be registered as well (by not specifying the /// second template argument). The interface should be registered to associate /// a nice name with the interface. -/// class RegisterAGBase : public PassInfo { public: RegisterAGBase(StringRef Name, const void *InterfaceID, @@ -196,27 +194,23 @@ struct RegisterAnalysisGroup : public RegisterAGBase { /// at runtime (which can be because of the RegisterPass constructors being run /// as the program starts up, or may be because a shared object just got /// loaded). -/// struct PassRegistrationListener { - PassRegistrationListener() {} - virtual ~PassRegistrationListener() {} + PassRegistrationListener() = default; + virtual ~PassRegistrationListener() = default; /// Callback functions - These functions are invoked whenever a pass is loaded /// or removed from the current executable. - /// virtual void passRegistered(const PassInfo *) {} /// enumeratePasses - Iterate over the registered passes, calling the /// passEnumerate callback on each PassInfo object. - /// void enumeratePasses(); /// passEnumerate - Callback function invoked when someone calls /// enumeratePasses on this PassRegistrationListener object. - /// virtual void passEnumerate(const PassInfo *) {} }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_PASSSUPPORT_H diff --git a/contrib/llvm/include/llvm/Passes/PassBuilder.h b/contrib/llvm/include/llvm/Passes/PassBuilder.h index 33433f6b4a10..b69988826253 100644 --- a/contrib/llvm/include/llvm/Passes/PassBuilder.h +++ b/contrib/llvm/include/llvm/Passes/PassBuilder.h @@ -29,10 +29,22 @@ class TargetMachine; /// A struct capturing PGO tunables. struct PGOOptions { - std::string ProfileGenFile = ""; - std::string ProfileUseFile = ""; - std::string SampleProfileFile = ""; - bool RunProfileGen = false; + PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "", + std::string SampleProfileFile = "", bool RunProfileGen = false, + bool SamplePGOSupport = false) + : ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile), + SampleProfileFile(SampleProfileFile), RunProfileGen(RunProfileGen), + SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) { + assert((RunProfileGen || + !SampleProfileFile.empty() || + !ProfileUseFile.empty() || + SamplePGOSupport) && "Illegal PGOOptions."); + } + std::string ProfileGenFile; + std::string ProfileUseFile; + std::string SampleProfileFile; + bool RunProfileGen; + bool SamplePGOSupport; }; /// \brief This class provides access to building LLVM's passes. @@ -59,6 +71,18 @@ public: std::vector<PipelineElement> InnerPipeline; }; + /// \brief ThinLTO phase. + /// + /// This enumerates the LLVM ThinLTO optimization phases. + enum class ThinLTOPhase { + /// No ThinLTO behavior needed. + None, + // ThinLTO prelink (summary) phase. + PreLink, + // ThinLTO postlink (backend compile) phase. + PostLink + }; + /// \brief LLVM-provided high-level optimization levels. /// /// This enumerates the LLVM-provided high-level optimization levels. Each @@ -202,13 +226,11 @@ public: /// require some transformations for semantic reasons, they should explicitly /// build them. /// - /// \p PrepareForThinLTO indicates whether this is invoked in - /// PrepareForThinLTO phase. Special handling is needed for sample PGO to - /// ensure profile accurate in the backend profile annotation phase. + /// \p Phase indicates the current ThinLTO phase. FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false, - bool PrepareForThinLTO = false); + ThinLTOPhase Phase, + bool DebugLogging = false); /// Construct the core LLVM module canonicalization and simplification /// pipeline. @@ -224,13 +246,11 @@ public: /// require some transformations for semantic reasons, they should explicitly /// build them. /// - /// \p PrepareForThinLTO indicates whether this is invoked in - /// PrepareForThinLTO phase. Special handling is needed for sample PGO to - /// ensure profile accurate in the backend profile annotation phase. + /// \p Phase indicates the current ThinLTO phase. ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false, - bool PrepareForThinLTO = false); + ThinLTOPhase Phase, + bool DebugLogging = false); /// Construct the core LLVM module optimization pipeline. /// diff --git a/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index fa9a87aed680..5a4098cf666c 100644 --- a/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -85,7 +85,7 @@ private: coveragemap_error Err; }; -/// \brief A Counter is an abstract value that describes how to compute the +/// A Counter is an abstract value that describes how to compute the /// execution count for a region of code using the collected profile count data. struct Counter { enum CounterKind { Zero, CounterValueReference, Expression }; @@ -125,23 +125,23 @@ public: return std::tie(LHS.Kind, LHS.ID) < std::tie(RHS.Kind, RHS.ID); } - /// \brief Return the counter that represents the number zero. + /// Return the counter that represents the number zero. static Counter getZero() { return Counter(); } - /// \brief Return the counter that corresponds to a specific profile counter. + /// Return the counter that corresponds to a specific profile counter. static Counter getCounter(unsigned CounterId) { return Counter(CounterValueReference, CounterId); } - /// \brief Return the counter that corresponds to a specific - /// addition counter expression. + /// Return the counter that corresponds to a specific addition counter + /// expression. static Counter getExpression(unsigned ExpressionId) { return Counter(Expression, ExpressionId); } }; -/// \brief A Counter expression is a value that represents an arithmetic -/// operation with two counters. +/// A Counter expression is a value that represents an arithmetic operation +/// with two counters. struct CounterExpression { enum ExprKind { Subtract, Add }; ExprKind Kind; @@ -151,17 +151,16 @@ struct CounterExpression { : Kind(Kind), LHS(LHS), RHS(RHS) {} }; -/// \brief A Counter expression builder is used to construct the -/// counter expressions. It avoids unnecessary duplication -/// and simplifies algebraic expressions. +/// A Counter expression builder is used to construct the counter expressions. +/// It avoids unnecessary duplication and simplifies algebraic expressions. class CounterExpressionBuilder { - /// \brief A list of all the counter expressions + /// A list of all the counter expressions std::vector<CounterExpression> Expressions; - /// \brief A lookup table for the index of a given expression. + /// A lookup table for the index of a given expression. DenseMap<CounterExpression, unsigned> ExpressionIndices; - /// \brief Return the counter which corresponds to the given expression. + /// Return the counter which corresponds to the given expression. /// /// If the given expression is already stored in the builder, a counter /// that references that expression is returned. Otherwise, the given @@ -177,44 +176,48 @@ class CounterExpressionBuilder { : CounterID(CounterID), Factor(Factor) {} }; - /// \brief Gather the terms of the expression tree for processing. + /// Gather the terms of the expression tree for processing. /// /// This collects each addition and subtraction referenced by the counter into /// a sequence that can be sorted and combined to build a simplified counter /// expression. void extractTerms(Counter C, int Sign, SmallVectorImpl<Term> &Terms); - /// \brief Simplifies the given expression tree + /// Simplifies the given expression tree /// by getting rid of algebraically redundant operations. Counter simplify(Counter ExpressionTree); public: ArrayRef<CounterExpression> getExpressions() const { return Expressions; } - /// \brief Return a counter that represents the expression - /// that adds LHS and RHS. + /// Return a counter that represents the expression that adds LHS and RHS. Counter add(Counter LHS, Counter RHS); - /// \brief Return a counter that represents the expression - /// that subtracts RHS from LHS. + /// Return a counter that represents the expression that subtracts RHS from + /// LHS. Counter subtract(Counter LHS, Counter RHS); }; -/// \brief A Counter mapping region associates a source range with -/// a specific counter. +using LineColPair = std::pair<unsigned, unsigned>; + +/// A Counter mapping region associates a source range with a specific counter. struct CounterMappingRegion { enum RegionKind { - /// \brief A CodeRegion associates some code with a counter + /// A CodeRegion associates some code with a counter CodeRegion, - /// \brief An ExpansionRegion represents a file expansion region that - /// associates a source range with the expansion of a virtual source file, - /// such as for a macro instantiation or #include file. + /// An ExpansionRegion represents a file expansion region that associates + /// a source range with the expansion of a virtual source file, such as + /// for a macro instantiation or #include file. ExpansionRegion, - /// \brief A SkippedRegion represents a source range with code that - /// was skipped by a preprocessor or similar means. - SkippedRegion + /// A SkippedRegion represents a source range with code that was skipped + /// by a preprocessor or similar means. + SkippedRegion, + + /// A GapRegion is like a CodeRegion, but its count is only set as the + /// line execution count when its the only region in the line. + GapRegion }; Counter Count; @@ -251,16 +254,21 @@ struct CounterMappingRegion { LineEnd, ColumnEnd, SkippedRegion); } - inline std::pair<unsigned, unsigned> startLoc() const { - return std::pair<unsigned, unsigned>(LineStart, ColumnStart); + static CounterMappingRegion + makeGapRegion(Counter Count, unsigned FileID, unsigned LineStart, + unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) { + return CounterMappingRegion(Count, FileID, 0, LineStart, ColumnStart, + LineEnd, (1U << 31) | ColumnEnd, GapRegion); } - inline std::pair<unsigned, unsigned> endLoc() const { - return std::pair<unsigned, unsigned>(LineEnd, ColumnEnd); + inline LineColPair startLoc() const { + return LineColPair(LineStart, ColumnStart); } + + inline LineColPair endLoc() const { return LineColPair(LineEnd, ColumnEnd); } }; -/// \brief Associates a source range with an execution count. +/// Associates a source range with an execution count. struct CountedRegion : public CounterMappingRegion { uint64_t ExecutionCount; @@ -268,8 +276,8 @@ struct CountedRegion : public CounterMappingRegion { : CounterMappingRegion(R), ExecutionCount(ExecutionCount) {} }; -/// \brief A Counter mapping context is used to connect the counters, -/// expressions and the obtained counter values. +/// A Counter mapping context is used to connect the counters, expressions +/// and the obtained counter values. class CounterMappingContext { ArrayRef<CounterExpression> Expressions; ArrayRef<uint64_t> CounterValues; @@ -284,20 +292,20 @@ public: void dump(const Counter &C, raw_ostream &OS) const; void dump(const Counter &C) const { dump(C, dbgs()); } - /// \brief Return the number of times that a region of code associated with - /// this counter was executed. + /// Return the number of times that a region of code associated with this + /// counter was executed. Expected<int64_t> evaluate(const Counter &C) const; }; -/// \brief Code coverage information for a single function. +/// Code coverage information for a single function. struct FunctionRecord { - /// \brief Raw function name. + /// Raw function name. std::string Name; - /// \brief Associated files. + /// Associated files. std::vector<std::string> Filenames; - /// \brief Regions in the function along with their counts. + /// Regions in the function along with their counts. std::vector<CountedRegion> CountedRegions; - /// \brief The number of times this function was executed. + /// The number of times this function was executed. uint64_t ExecutionCount; FunctionRecord(StringRef Name, ArrayRef<StringRef> Filenames) @@ -313,7 +321,7 @@ struct FunctionRecord { } }; -/// \brief Iterator over Functions, optionally filtered to a single file. +/// Iterator over Functions, optionally filtered to a single file. class FunctionRecordIterator : public iterator_facade_base<FunctionRecordIterator, std::forward_iterator_tag, FunctionRecord> { @@ -321,7 +329,7 @@ class FunctionRecordIterator ArrayRef<FunctionRecord>::iterator Current; StringRef Filename; - /// \brief Skip records whose primary file is not \c Filename. + /// Skip records whose primary file is not \c Filename. void skipOtherFiles(); public: @@ -347,17 +355,17 @@ public: } }; -/// \brief Coverage information for a macro expansion or #included file. +/// Coverage information for a macro expansion or #included file. /// /// When covered code has pieces that can be expanded for more detail, such as a /// preprocessor macro use and its definition, these are represented as /// expansions whose coverage can be looked up independently. struct ExpansionRecord { - /// \brief The abstract file this expansion covers. + /// The abstract file this expansion covers. unsigned FileID; - /// \brief The region that expands to this record. + /// The region that expands to this record. const CountedRegion &Region; - /// \brief Coverage for the expansion. + /// Coverage for the expansion. const FunctionRecord &Function; ExpansionRecord(const CountedRegion &Region, @@ -365,38 +373,99 @@ struct ExpansionRecord { : FileID(Region.ExpandedFileID), Region(Region), Function(Function) {} }; -/// \brief The execution count information starting at a point in a file. +/// The execution count information starting at a point in a file. /// /// A sequence of CoverageSegments gives execution counts for a file in format /// that's simple to iterate through for processing. struct CoverageSegment { - /// \brief The line where this segment begins. + /// The line where this segment begins. unsigned Line; - /// \brief The column where this segment begins. + /// The column where this segment begins. unsigned Col; - /// \brief The execution count, or zero if no count was recorded. + /// The execution count, or zero if no count was recorded. uint64_t Count; - /// \brief When false, the segment was uninstrumented or skipped. + /// When false, the segment was uninstrumented or skipped. bool HasCount; - /// \brief Whether this enters a new region or returns to a previous count. + /// Whether this enters a new region or returns to a previous count. bool IsRegionEntry; + /// Whether this enters a gap region. + bool IsGapRegion; CoverageSegment(unsigned Line, unsigned Col, bool IsRegionEntry) : Line(Line), Col(Col), Count(0), HasCount(false), - IsRegionEntry(IsRegionEntry) {} + IsRegionEntry(IsRegionEntry), IsGapRegion(false) {} CoverageSegment(unsigned Line, unsigned Col, uint64_t Count, - bool IsRegionEntry) + bool IsRegionEntry, bool IsGapRegion = false) : Line(Line), Col(Col), Count(Count), HasCount(true), - IsRegionEntry(IsRegionEntry) {} + IsRegionEntry(IsRegionEntry), IsGapRegion(IsGapRegion) {} friend bool operator==(const CoverageSegment &L, const CoverageSegment &R) { - return std::tie(L.Line, L.Col, L.Count, L.HasCount, L.IsRegionEntry) == - std::tie(R.Line, R.Col, R.Count, R.HasCount, R.IsRegionEntry); + return std::tie(L.Line, L.Col, L.Count, L.HasCount, L.IsRegionEntry, + L.IsGapRegion) == std::tie(R.Line, R.Col, R.Count, + R.HasCount, R.IsRegionEntry, + R.IsGapRegion); } }; -/// \brief Coverage information to be processed or displayed. +/// An instantiation group contains a \c FunctionRecord list, such that each +/// record corresponds to a distinct instantiation of the same function. +/// +/// Note that it's possible for a function to have more than one instantiation +/// (consider C++ template specializations or static inline functions). +class InstantiationGroup { + friend class CoverageMapping; + + unsigned Line; + unsigned Col; + std::vector<const FunctionRecord *> Instantiations; + + InstantiationGroup(unsigned Line, unsigned Col, + std::vector<const FunctionRecord *> Instantiations) + : Line(Line), Col(Col), Instantiations(std::move(Instantiations)) {} + +public: + InstantiationGroup(const InstantiationGroup &) = delete; + InstantiationGroup(InstantiationGroup &&) = default; + + /// Get the number of instantiations in this group. + size_t size() const { return Instantiations.size(); } + + /// Get the line where the common function was defined. + unsigned getLine() const { return Line; } + + /// Get the column where the common function was defined. + unsigned getColumn() const { return Col; } + + /// Check if the instantiations in this group have a common mangled name. + bool hasName() const { + for (unsigned I = 1, E = Instantiations.size(); I < E; ++I) + if (Instantiations[I]->Name != Instantiations[0]->Name) + return false; + return true; + } + + /// Get the common mangled name for instantiations in this group. + StringRef getName() const { + assert(hasName() && "Instantiations don't have a shared name"); + return Instantiations[0]->Name; + } + + /// Get the total execution count of all instantiations in this group. + uint64_t getTotalExecutionCount() const { + uint64_t Count = 0; + for (const FunctionRecord *F : Instantiations) + Count += F->ExecutionCount; + return Count; + } + + /// Get the instantiations in this group. + ArrayRef<const FunctionRecord *> getInstantiations() const { + return Instantiations; + } +}; + +/// Coverage information to be processed or displayed. /// /// This represents the coverage of an entire file, expansion, or function. It /// provides a sequence of CoverageSegments to iterate through, as well as the @@ -413,9 +482,11 @@ public: CoverageData(StringRef Filename) : Filename(Filename) {} - /// \brief Get the name of the file this data covers. + /// Get the name of the file this data covers. StringRef getFilename() const { return Filename; } + /// Get an iterator over the coverage segments for this object. The segments + /// are guaranteed to be uniqued and sorted by location. std::vector<CoverageSegment>::const_iterator begin() const { return Segments.begin(); } @@ -426,22 +497,23 @@ public: bool empty() const { return Segments.empty(); } - /// \brief Expansions that can be further processed. + /// Expansions that can be further processed. ArrayRef<ExpansionRecord> getExpansions() const { return Expansions; } }; -/// \brief The mapping of profile information to coverage data. +/// The mapping of profile information to coverage data. /// /// This is the main interface to get coverage information, using a profile to /// fill out execution counts. class CoverageMapping { StringSet<> FunctionNames; std::vector<FunctionRecord> Functions; - unsigned MismatchedFunctionCount = 0; + std::vector<std::pair<std::string, uint64_t>> FuncHashMismatches; + std::vector<std::pair<std::string, uint64_t>> FuncCounterMismatches; CoverageMapping() = default; - /// \brief Add a function record corresponding to \p Record. + /// Add a function record corresponding to \p Record. Error loadFunctionRecord(const CoverageMappingRecord &Record, IndexedInstrProfReader &ProfileReader); @@ -449,59 +521,162 @@ public: CoverageMapping(const CoverageMapping &) = delete; CoverageMapping &operator=(const CoverageMapping &) = delete; - /// \brief Load the coverage mapping using the given readers. + /// Load the coverage mapping using the given readers. static Expected<std::unique_ptr<CoverageMapping>> load(ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders, IndexedInstrProfReader &ProfileReader); + /// Load the coverage mapping from the given object files and profile. If + /// \p Arches is non-empty, it must specify an architecture for each object. static Expected<std::unique_ptr<CoverageMapping>> load(ArrayRef<StringRef> ObjectFilenames, StringRef ProfileFilename, - StringRef Arch = StringRef()); + ArrayRef<StringRef> Arches = None); - /// \brief The number of functions that couldn't have their profiles mapped. + /// The number of functions that couldn't have their profiles mapped. /// /// This is a count of functions whose profile is out of date or otherwise /// can't be associated with any coverage information. - unsigned getMismatchedCount() { return MismatchedFunctionCount; } + unsigned getMismatchedCount() const { + return FuncHashMismatches.size() + FuncCounterMismatches.size(); + } + + /// A hash mismatch occurs when a profile record for a symbol does not have + /// the same hash as a coverage mapping record for the same symbol. This + /// returns a list of hash mismatches, where each mismatch is a pair of the + /// symbol name and its coverage mapping hash. + ArrayRef<std::pair<std::string, uint64_t>> getHashMismatches() const { + return FuncHashMismatches; + } - /// \brief Returns a lexicographically sorted, unique list of files that are + /// A counter mismatch occurs when there is an error when evaluating the + /// counter expressions in a coverage mapping record. This returns a list of + /// counter mismatches, where each mismatch is a pair of the symbol name and + /// the number of valid evaluated counter expressions. + ArrayRef<std::pair<std::string, uint64_t>> getCounterMismatches() const { + return FuncCounterMismatches; + } + + /// Returns a lexicographically sorted, unique list of files that are /// covered. std::vector<StringRef> getUniqueSourceFiles() const; - /// \brief Get the coverage for a particular file. + /// Get the coverage for a particular file. /// /// The given filename must be the name as recorded in the coverage /// information. That is, only names returned from getUniqueSourceFiles will /// yield a result. CoverageData getCoverageForFile(StringRef Filename) const; - /// \brief Gets all of the functions covered by this profile. + /// Get the coverage for a particular function. + CoverageData getCoverageForFunction(const FunctionRecord &Function) const; + + /// Get the coverage for an expansion within a coverage set. + CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion) const; + + /// Gets all of the functions covered by this profile. iterator_range<FunctionRecordIterator> getCoveredFunctions() const { return make_range(FunctionRecordIterator(Functions), FunctionRecordIterator()); } - /// \brief Gets all of the functions in a particular file. + /// Gets all of the functions in a particular file. iterator_range<FunctionRecordIterator> getCoveredFunctions(StringRef Filename) const { return make_range(FunctionRecordIterator(Functions, Filename), FunctionRecordIterator()); } - /// \brief Get the list of function instantiations in the file. + /// Get the list of function instantiation groups in a particular file. /// - /// Functions that are instantiated more than once, such as C++ template - /// specializations, have distinct coverage records for each instantiation. - std::vector<const FunctionRecord *> - getInstantiations(StringRef Filename) const; + /// Every instantiation group in a program is attributed to exactly one file: + /// the file in which the definition for the common function begins. + std::vector<InstantiationGroup> + getInstantiationGroups(StringRef Filename) const; +}; - /// \brief Get the coverage for a particular function. - CoverageData getCoverageForFunction(const FunctionRecord &Function) const; +/// Coverage statistics for a single line. +class LineCoverageStats { + uint64_t ExecutionCount; + bool HasMultipleRegions; + bool Mapped; + unsigned Line; + ArrayRef<const CoverageSegment *> LineSegments; + const CoverageSegment *WrappedSegment; - /// \brief Get the coverage for an expansion within a coverage set. - CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion) const; + friend class LineCoverageIterator; + LineCoverageStats() = default; + +public: + LineCoverageStats(ArrayRef<const CoverageSegment *> LineSegments, + const CoverageSegment *WrappedSegment, unsigned Line); + + uint64_t getExecutionCount() const { return ExecutionCount; } + + bool hasMultipleRegions() const { return HasMultipleRegions; } + + bool isMapped() const { return Mapped; } + + unsigned getLine() const { return Line; } + + ArrayRef<const CoverageSegment *> getLineSegments() const { + return LineSegments; + } + + const CoverageSegment *getWrappedSegment() const { return WrappedSegment; } +}; + +/// An iterator over the \c LineCoverageStats objects for lines described by +/// a \c CoverageData instance. +class LineCoverageIterator + : public iterator_facade_base< + LineCoverageIterator, std::forward_iterator_tag, LineCoverageStats> { +public: + LineCoverageIterator(const CoverageData &CD) + : LineCoverageIterator(CD, CD.begin()->Line) {} + + LineCoverageIterator(const CoverageData &CD, unsigned Line) + : CD(CD), WrappedSegment(nullptr), Next(CD.begin()), Ended(false), + Line(Line), Segments(), Stats() { + this->operator++(); + } + + LineCoverageIterator &operator=(const LineCoverageIterator &R) = default; + + bool operator==(const LineCoverageIterator &R) const { + return &CD == &R.CD && Next == R.Next && Ended == R.Ended; + } + + const LineCoverageStats &operator*() const { return Stats; } + + LineCoverageStats &operator*() { return Stats; } + + LineCoverageIterator &operator++(); + + LineCoverageIterator getEnd() const { + auto EndIt = *this; + EndIt.Next = CD.end(); + EndIt.Ended = true; + return EndIt; + } + +private: + const CoverageData &CD; + const CoverageSegment *WrappedSegment; + std::vector<CoverageSegment>::const_iterator Next; + bool Ended; + unsigned Line; + SmallVector<const CoverageSegment *, 4> Segments; + LineCoverageStats Stats; }; +/// Get a \c LineCoverageIterator range for the lines described by \p CD. +static inline iterator_range<LineCoverageIterator> +getLineCoverageStats(const coverage::CoverageData &CD) { + auto Begin = LineCoverageIterator(CD); + auto End = Begin.getEnd(); + return make_range(Begin, End); +} + // Profile coverage map has the following layout: // [CoverageMapFileHeader] // [ArrayStart] @@ -602,7 +777,10 @@ enum CovMapVersion { // name string pointer to MD5 to support name section compression. Name // section is also compressed. Version2 = 1, - // The current version is Version2 + // A new interpretation of the columnEnd field is added in order to mark + // regions as gap areas. + Version3 = 2, + // The current version is Version3 CurrentVersion = INSTR_PROF_COVMAP_VERSION }; @@ -618,7 +796,7 @@ template <class IntPtrT> struct CovMapTraits<CovMapVersion::Version1, IntPtrT> { } // end namespace coverage -/// \brief Provide DenseMapInfo for CounterExpression +/// Provide DenseMapInfo for CounterExpression template<> struct DenseMapInfo<coverage::CounterExpression> { static inline coverage::CounterExpression getEmptyKey() { using namespace coverage; diff --git a/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 5b372252a9ac..633e51565cd2 100644 --- a/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/contrib/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -44,18 +44,26 @@ struct CoverageMappingRecord { /// \brief A file format agnostic iterator over coverage mapping data. class CoverageMappingIterator : public std::iterator<std::input_iterator_tag, CoverageMappingRecord> { - CoverageMappingReader *Reader = nullptr; + CoverageMappingReader *Reader; CoverageMappingRecord Record; + coveragemap_error ReadErr; void increment(); public: - CoverageMappingIterator() = default; + CoverageMappingIterator() + : Reader(nullptr), Record(), ReadErr(coveragemap_error::success) {} - CoverageMappingIterator(CoverageMappingReader *Reader) : Reader(Reader) { + CoverageMappingIterator(CoverageMappingReader *Reader) + : Reader(Reader), Record(), ReadErr(coveragemap_error::success) { increment(); } + ~CoverageMappingIterator() { + if (ReadErr != coveragemap_error::success) + llvm_unreachable("Unexpected error in coverage mapping iterator"); + } + CoverageMappingIterator &operator++() { increment(); return *this; @@ -66,8 +74,22 @@ public: bool operator!=(const CoverageMappingIterator &RHS) { return Reader != RHS.Reader; } - CoverageMappingRecord &operator*() { return Record; } - CoverageMappingRecord *operator->() { return &Record; } + Expected<CoverageMappingRecord &> operator*() { + if (ReadErr != coveragemap_error::success) { + auto E = make_error<CoverageMapError>(ReadErr); + ReadErr = coveragemap_error::success; + return std::move(E); + } + return Record; + } + Expected<CoverageMappingRecord *> operator->() { + if (ReadErr != coveragemap_error::success) { + auto E = make_error<CoverageMapError>(ReadErr); + ReadErr = coveragemap_error::success; + return std::move(E); + } + return &Record; + } }; class CoverageMappingReader { diff --git a/contrib/llvm/include/llvm/Support/GCOV.h b/contrib/llvm/include/llvm/ProfileData/GCOV.h index 02016e7dbd62..497f80b87b26 100644 --- a/contrib/llvm/include/llvm/Support/GCOV.h +++ b/contrib/llvm/include/llvm/ProfileData/GCOV.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_GCOV_H -#define LLVM_SUPPORT_GCOV_H +#ifndef LLVM_PROFILEDATA_GCOV_H +#define LLVM_PROFILEDATA_GCOV_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProf.h b/contrib/llvm/include/llvm/ProfileData/InstrProf.h index 772187f70153..b08b78cd593c 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProf.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProf.h @@ -295,7 +295,8 @@ enum class instrprof_error { value_site_count_mismatch, compress_failed, uncompress_failed, - empty_raw_profile + empty_raw_profile, + zlib_unavailable }; inline std::error_code make_error_code(instrprof_error E) { @@ -858,7 +859,9 @@ enum ProfVersion { // In this version, profile summary data \c IndexedInstrProf::Summary is // stored after the profile header. Version4 = 4, - // The current version is 4. + // In this version, the frontend PGO stable hash algorithm defaults to V2. + Version5 = 5, + // The current version is 5. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfData.inc b/contrib/llvm/include/llvm/ProfileData/InstrProfData.inc index be0dd4ad04bf..6a98dc7b9b85 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -628,9 +628,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 4 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 4 +#define INSTR_PROF_INDEX_VERSION 5 /* Coverage mapping format vresion (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 1 +#define INSTR_PROF_COVMAP_VERSION 2 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 diff --git a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h index 424360e0f765..aa58ead1eda1 100644 --- a/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/contrib/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -397,6 +397,8 @@ private: std::unique_ptr<InstrProfReaderIndexBase> Index; /// Profile summary data. std::unique_ptr<ProfileSummary> Summary; + // Index to the current record in the record array. + unsigned RecordIndex; // Read the profile summary. Return a pointer pointing to one byte past the // end of the summary data if it exists or the input \c Cur. @@ -405,7 +407,7 @@ private: public: IndexedInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer) - : DataBuffer(std::move(DataBuffer)) {} + : DataBuffer(std::move(DataBuffer)), RecordIndex(0) {} IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; diff --git a/contrib/llvm/include/llvm/ProfileData/SampleProf.h b/contrib/llvm/include/llvm/ProfileData/SampleProf.h index 7fc258831be8..9eccafc65f3a 100644 --- a/contrib/llvm/include/llvm/ProfileData/SampleProf.h +++ b/contrib/llvm/include/llvm/ProfileData/SampleProf.h @@ -185,7 +185,9 @@ raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample); class FunctionSamples; using BodySampleMap = std::map<LineLocation, SampleRecord>; -using FunctionSamplesMap = StringMap<FunctionSamples>; +// NOTE: Using a StringMap here makes parsed profiles consume around 17% more +// memory, which is *very* significant for large profiles. +using FunctionSamplesMap = std::map<std::string, FunctionSamples>; using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>; /// Representation of the samples collected for a function. @@ -278,7 +280,7 @@ public: return nullptr; auto FS = iter->second.find(CalleeName); if (FS != iter->second.end()) - return &FS->getValue(); + return &FS->second; // If we cannot find exact match of the callee name, return the FS with // the max total count. uint64_t MaxTotalSamples = 0; @@ -296,10 +298,33 @@ public: /// Return the total number of samples collected inside the function. uint64_t getTotalSamples() const { return TotalSamples; } - /// Return the total number of samples collected at the head of the - /// function. + /// Return the total number of branch samples that have the function as the + /// branch target. This should be equivalent to the sample of the first + /// instruction of the symbol. But as we directly get this info for raw + /// profile without referring to potentially inaccurate debug info, this + /// gives more accurate profile data and is preferred for standalone symbols. uint64_t getHeadSamples() const { return TotalHeadSamples; } + /// Return the sample count of the first instruction of the function. + /// The function can be either a standalone symbol or an inlined function. + uint64_t getEntrySamples() const { + // Use either BodySamples or CallsiteSamples which ever has the smaller + // lineno. + if (!BodySamples.empty() && + (CallsiteSamples.empty() || + BodySamples.begin()->first < CallsiteSamples.begin()->first)) + return BodySamples.begin()->second.getSamples(); + if (!CallsiteSamples.empty()) { + uint64_t T = 0; + // An indirect callsite may be promoted to several inlined direct calls. + // We need to get the sum of them. + for (const auto &N_FS : CallsiteSamples.begin()->second) + T += N_FS.second.getEntrySamples(); + return T; + } + return 0; + } + /// Return all the samples collected in the body of the function. const BodySampleMap &getBodySamples() const { return BodySamples; } @@ -324,24 +349,32 @@ public: const LineLocation &Loc = I.first; FunctionSamplesMap &FSMap = functionSamplesAt(Loc); for (const auto &Rec : I.second) - MergeResult(Result, FSMap[Rec.first()].merge(Rec.second, Weight)); + MergeResult(Result, FSMap[Rec.first].merge(Rec.second, Weight)); } return Result; } - /// Recursively traverses all children, if the corresponding function is - /// not defined in module \p M, and its total sample is no less than - /// \p Threshold, add its corresponding GUID to \p S. - void findImportedFunctions(DenseSet<GlobalValue::GUID> &S, const Module *M, - uint64_t Threshold) const { + /// Recursively traverses all children, if the total sample count of the + /// corresponding function is no less than \p Threshold, add its corresponding + /// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID + /// to \p S. + void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S, const Module *M, + uint64_t Threshold) const { if (TotalSamples <= Threshold) return; - Function *F = M->getFunction(Name); - if (!F || !F->getSubprogram()) - S.insert(Function::getGUID(Name)); - for (auto CS : CallsiteSamples) + S.insert(Function::getGUID(Name)); + // Import hot CallTargets, which may not be available in IR because full + // profile annotation cannot be done until backend compilation in ThinLTO. + for (const auto &BS : BodySamples) + for (const auto &TS : BS.second.getCallTargets()) + if (TS.getValue() > Threshold) { + Function *Callee = M->getFunction(TS.getKey()); + if (!Callee || !Callee->getSubprogram()) + S.insert(Function::getGUID(TS.getKey())); + } + for (const auto &CS : CallsiteSamples) for (const auto &NameFS : CS.second) - NameFS.second.findImportedFunctions(S, M, Threshold); + NameFS.second.findInlinedFunctions(S, M, Threshold); } /// Set the name of the function. diff --git a/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h b/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h index 9c1f357cbbd1..0e9ab2dc60ee 100644 --- a/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/contrib/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -217,10 +217,10 @@ #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/ProfileSummary.h" +#include "llvm/ProfileData/GCOV.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" -#include "llvm/Support/GCOV.h" #include "llvm/Support/MemoryBuffer.h" #include <algorithm> #include <cstdint> diff --git a/contrib/llvm/include/llvm/Support/AArch64TargetParser.def b/contrib/llvm/include/llvm/Support/AArch64TargetParser.def index 09f9602a24d9..30c7924ea5f1 100644 --- a/contrib/llvm/include/llvm/Support/AArch64TargetParser.def +++ b/contrib/llvm/include/llvm/Support/AArch64TargetParser.def @@ -16,19 +16,25 @@ #ifndef AARCH64_ARCH #define AARCH64_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) #endif -AARCH64_ARCH("invalid", AK_INVALID, nullptr, nullptr, +AARCH64_ARCH("invalid", INVALID, "", "", ARMBuildAttrs::CPUArch::v8_A, FK_NONE, AArch64::AEK_NONE) -AARCH64_ARCH("armv8-a", AK_ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8_A, +AARCH64_ARCH("armv8-a", ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (AArch64::AEK_CRYPTO | AArch64::AEK_FP | AArch64::AEK_SIMD)) -AARCH64_ARCH("armv8.1-a", AK_ARMV8_1A, "8.1-A", "v8.1a", +AARCH64_ARCH("armv8.1-a", ARMV8_1A, "8.1-A", "v8.1a", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP | - AArch64::AEK_SIMD | AArch64::AEK_LSE)) -AARCH64_ARCH("armv8.2-a", AK_ARMV8_2A, "8.2-A", "v8.2a", + AArch64::AEK_SIMD | AArch64::AEK_LSE | AArch64::AEK_RDM)) +AARCH64_ARCH("armv8.2-a", ARMV8_2A, "8.2-A", "v8.2a", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP | - AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE)) + AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE | + AArch64::AEK_RDM)) +AARCH64_ARCH("armv8.3-a", ARMV8_3A, "8.3-A", "v8.3a", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP | + AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE | + AArch64::AEK_RDM | AArch64::AEK_RCPC)) #undef AARCH64_ARCH #ifndef AARCH64_ARCH_EXT_NAME @@ -39,50 +45,59 @@ AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") +AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") +AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") +AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") #undef AARCH64_ARCH_EXT_NAME #ifndef AARCH64_CPU_NAME #define AARCH64_CPU_NAME(NAME, ID, DEFAULT_FPU, IS_DEFAULT, DEFAULT_EXT) #endif -AARCH64_CPU_NAME("cortex-a35", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("cortex-a53", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, true, +AARCH64_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, true, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("cortex-a57", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("cortex-a55", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC)) +AARCH64_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("cortex-a72", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("cortex-a73", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("cyclone", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("cortex-a75", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC)) +AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) -AARCH64_CPU_NAME("exynos-m1", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("exynos-m2", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("exynos-m3", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("falkor", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("kryo", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("falkor", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_CRC | AArch64::AEK_RDM)) +AARCH64_CPU_NAME("saphira", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_PROFILE)) +AARCH64_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("thunderx2t99", AK_ARMV8_1A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("thunderx2t99", ARMV8_1A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) -AARCH64_CPU_NAME("thunderx", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("thunderx", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) -AARCH64_CPU_NAME("thunderxt88", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("thunderxt88", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) -AARCH64_CPU_NAME("thunderxt81", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("thunderxt81", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) -AARCH64_CPU_NAME("thunderxt83", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, +AARCH64_CPU_NAME("thunderxt83", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) // Invalid CPU -AARCH64_CPU_NAME("invalid", AK_INVALID, FK_INVALID, true, AArch64::AEK_INVALID) +AARCH64_CPU_NAME("invalid", INVALID, FK_INVALID, true, AArch64::AEK_INVALID) #undef AARCH64_CPU_NAME diff --git a/contrib/llvm/include/llvm/Support/AMDGPUKernelDescriptor.h b/contrib/llvm/include/llvm/Support/AMDGPUKernelDescriptor.h new file mode 100644 index 000000000000..ce2c0c1c959e --- /dev/null +++ b/contrib/llvm/include/llvm/Support/AMDGPUKernelDescriptor.h @@ -0,0 +1,139 @@ +//===--- AMDGPUKernelDescriptor.h -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file +/// \brief AMDGPU kernel descriptor definitions. For more information, visit +/// https://llvm.org/docs/AMDGPUUsage.html#kernel-descriptor-for-gfx6-gfx9 +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_AMDGPUKERNELDESCRIPTOR_H +#define LLVM_SUPPORT_AMDGPUKERNELDESCRIPTOR_H + +#include <cstdint> + +// Creates enumeration entries used for packing bits into integers. Enumeration +// entries include bit shift amount, bit width, and bit mask. +#define AMDGPU_BITS_ENUM_ENTRY(name, shift, width) \ + name ## _SHIFT = (shift), \ + name ## _WIDTH = (width), \ + name = (((1 << (width)) - 1) << (shift)) \ + +// Gets bits for specified bit mask from specified source. +#define AMDGPU_BITS_GET(src, mask) \ + ((src & mask) >> mask ## _SHIFT) \ + +// Sets bits for specified bit mask in specified destination. +#define AMDGPU_BITS_SET(dst, mask, val) \ + dst &= (~(1 << mask ## _SHIFT) & ~mask); \ + dst |= (((val) << mask ## _SHIFT) & mask) \ + +namespace llvm { +namespace AMDGPU { +namespace HSAKD { + +/// \brief Floating point rounding modes. +enum : uint8_t { + AMDGPU_FLOAT_ROUND_MODE_NEAR_EVEN = 0, + AMDGPU_FLOAT_ROUND_MODE_PLUS_INFINITY = 1, + AMDGPU_FLOAT_ROUND_MODE_MINUS_INFINITY = 2, + AMDGPU_FLOAT_ROUND_MODE_ZERO = 3, +}; + +/// \brief Floating point denorm modes. +enum : uint8_t { + AMDGPU_FLOAT_DENORM_MODE_FLUSH_SRC_DST = 0, + AMDGPU_FLOAT_DENORM_MODE_FLUSH_DST = 1, + AMDGPU_FLOAT_DENORM_MODE_FLUSH_SRC = 2, + AMDGPU_FLOAT_DENORM_MODE_FLUSH_NONE = 3, +}; + +/// \brief System VGPR workitem IDs. +enum : uint8_t { + AMDGPU_SYSTEM_VGPR_WORKITEM_ID_X = 0, + AMDGPU_SYSTEM_VGPR_WORKITEM_ID_X_Y = 1, + AMDGPU_SYSTEM_VGPR_WORKITEM_ID_X_Y_Z = 2, + AMDGPU_SYSTEM_VGPR_WORKITEM_ID_UNDEFINED = 3, +}; + +/// \brief Compute program resource register one layout. +enum ComputePgmRsrc1 { + AMDGPU_BITS_ENUM_ENTRY(GRANULATED_WORKITEM_VGPR_COUNT, 0, 6), + AMDGPU_BITS_ENUM_ENTRY(GRANULATED_WAVEFRONT_SGPR_COUNT, 6, 4), + AMDGPU_BITS_ENUM_ENTRY(PRIORITY, 10, 2), + AMDGPU_BITS_ENUM_ENTRY(FLOAT_ROUND_MODE_32, 12, 2), + AMDGPU_BITS_ENUM_ENTRY(FLOAT_ROUND_MODE_16_64, 14, 2), + AMDGPU_BITS_ENUM_ENTRY(FLOAT_DENORM_MODE_32, 16, 2), + AMDGPU_BITS_ENUM_ENTRY(FLOAT_DENORM_MODE_16_64, 18, 2), + AMDGPU_BITS_ENUM_ENTRY(PRIV, 20, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_DX10_CLAMP, 21, 1), + AMDGPU_BITS_ENUM_ENTRY(DEBUG_MODE, 22, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_IEEE_MODE, 23, 1), + AMDGPU_BITS_ENUM_ENTRY(BULKY, 24, 1), + AMDGPU_BITS_ENUM_ENTRY(CDBG_USER, 25, 1), + AMDGPU_BITS_ENUM_ENTRY(FP16_OVFL, 26, 1), + AMDGPU_BITS_ENUM_ENTRY(RESERVED0, 27, 5), +}; + +/// \brief Compute program resource register two layout. +enum ComputePgmRsrc2 { + AMDGPU_BITS_ENUM_ENTRY(ENABLE_SGPR_PRIVATE_SEGMENT_WAVE_OFFSET, 0, 1), + AMDGPU_BITS_ENUM_ENTRY(USER_SGPR_COUNT, 1, 5), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_TRAP_HANDLER, 6, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_SGPR_WORKGROUP_ID_X, 7, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_SGPR_WORKGROUP_ID_Y, 8, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_SGPR_WORKGROUP_ID_Z, 9, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_SGPR_WORKGROUP_INFO, 10, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_VGPR_WORKITEM_ID, 11, 2), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_ADDRESS_WATCH, 13, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_MEMORY, 14, 1), + AMDGPU_BITS_ENUM_ENTRY(GRANULATED_LDS_SIZE, 15, 9), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, 24, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, 25, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, 26, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, 27, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, 28, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, 29, 1), + AMDGPU_BITS_ENUM_ENTRY(ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, 30, 1), + AMDGPU_BITS_ENUM_ENTRY(RESERVED1, 31, 1), +}; + +/// \brief Kernel descriptor layout. This layout should be kept backwards +/// compatible as it is consumed by the command processor. +struct KernelDescriptor final { + uint32_t GroupSegmentFixedSize; + uint32_t PrivateSegmentFixedSize; + uint32_t MaxFlatWorkGroupSize; + uint64_t IsDynamicCallStack : 1; + uint64_t IsXNACKEnabled : 1; + uint64_t Reserved0 : 30; + int64_t KernelCodeEntryByteOffset; + uint64_t Reserved1[3]; + uint32_t ComputePgmRsrc1; + uint32_t ComputePgmRsrc2; + uint64_t EnableSGPRPrivateSegmentBuffer : 1; + uint64_t EnableSGPRDispatchPtr : 1; + uint64_t EnableSGPRQueuePtr : 1; + uint64_t EnableSGPRKernargSegmentPtr : 1; + uint64_t EnableSGPRDispatchID : 1; + uint64_t EnableSGPRFlatScratchInit : 1; + uint64_t EnableSGPRPrivateSegmentSize : 1; + uint64_t EnableSGPRGridWorkgroupCountX : 1; + uint64_t EnableSGPRGridWorkgroupCountY : 1; + uint64_t EnableSGPRGridWorkgroupCountZ : 1; + uint64_t Reserved2 : 54; + + KernelDescriptor() = default; +}; + +} // end namespace HSAKD +} // end namespace AMDGPU +} // end namespace llvm + +#endif // LLVM_SUPPORT_AMDGPUKERNELDESCRIPTOR_H diff --git a/contrib/llvm/include/llvm/Support/AMDGPUCodeObjectMetadata.h b/contrib/llvm/include/llvm/Support/AMDGPUMetadata.h index d274c5ee9184..00039a75c51d 100644 --- a/contrib/llvm/include/llvm/Support/AMDGPUCodeObjectMetadata.h +++ b/contrib/llvm/include/llvm/Support/AMDGPUMetadata.h @@ -1,4 +1,4 @@ -//===--- AMDGPUCodeObjectMetadata.h -----------------------------*- C++ -*-===// +//===--- AMDGPUMetadata.h ---------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -8,14 +8,13 @@ //===----------------------------------------------------------------------===// // /// \file -/// \brief AMDGPU Code Object Metadata definitions and in-memory -/// representations. +/// \brief AMDGPU metadata definitions and in-memory representations. /// // //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_AMDGPUCODEOBJECTMETADATA_H -#define LLVM_SUPPORT_AMDGPUCODEOBJECTMETADATA_H +#ifndef LLVM_SUPPORT_AMDGPUMETADATA_H +#define LLVM_SUPPORT_AMDGPUMETADATA_H #include <cstdint> #include <string> @@ -26,21 +25,19 @@ namespace llvm { namespace AMDGPU { //===----------------------------------------------------------------------===// -// Code Object Metadata. +// HSA metadata. //===----------------------------------------------------------------------===// -namespace CodeObject { +namespace HSAMD { -/// \brief Code object metadata major version. -constexpr uint32_t MetadataVersionMajor = 1; -/// \brief Code object metadata minor version. -constexpr uint32_t MetadataVersionMinor = 0; +/// \brief HSA metadata major version. +constexpr uint32_t VersionMajor = 1; +/// \brief HSA metadata minor version. +constexpr uint32_t VersionMinor = 0; -/// \brief Code object metadata beginning assembler directive. -constexpr char MetadataAssemblerDirectiveBegin[] = - ".amdgpu_code_object_metadata"; -/// \brief Code object metadata ending assembler directive. -constexpr char MetadataAssemblerDirectiveEnd[] = - ".end_amdgpu_code_object_metadata"; +/// \brief HSA metadata beginning assembler directive. +constexpr char AssemblerDirectiveBegin[] = ".amd_amdgpu_hsa_metadata"; +/// \brief HSA metadata ending assembler directive. +constexpr char AssemblerDirectiveEnd[] = ".end_amd_amdgpu_hsa_metadata"; /// \brief Access qualifiers. enum class AccessQualifier : uint8_t { @@ -115,6 +112,8 @@ constexpr char ReqdWorkGroupSize[] = "ReqdWorkGroupSize"; constexpr char WorkGroupSizeHint[] = "WorkGroupSizeHint"; /// \brief Key for Kernel::Attr::Metadata::mVecTypeHint. constexpr char VecTypeHint[] = "VecTypeHint"; +/// \brief Key for Kernel::Attr::Metadata::mRuntimeHandle. +constexpr char RuntimeHandle[] = "RuntimeHandle"; } // end namespace Key /// \brief In-memory representation of kernel attributes metadata. @@ -125,20 +124,22 @@ struct Metadata final { std::vector<uint32_t> mWorkGroupSizeHint = std::vector<uint32_t>(); /// \brief 'vec_type_hint' attribute. Optional. std::string mVecTypeHint = std::string(); + /// \brief External symbol created by runtime to store the kernel address + /// for enqueued blocks. + std::string mRuntimeHandle = std::string(); /// \brief Default constructor. Metadata() = default; /// \returns True if kernel attributes metadata is empty, false otherwise. bool empty() const { - return mReqdWorkGroupSize.empty() && - mWorkGroupSizeHint.empty() && - mVecTypeHint.empty(); + return !notEmpty(); } /// \returns True if kernel attributes metadata is not empty, false otherwise. bool notEmpty() const { - return !empty(); + return !mReqdWorkGroupSize.empty() || !mWorkGroupSizeHint.empty() || + !mVecTypeHint.empty() || !mRuntimeHandle.empty(); } }; @@ -150,6 +151,10 @@ struct Metadata final { namespace Arg { namespace Key { +/// \brief Key for Kernel::Arg::Metadata::mName. +constexpr char Name[] = "Name"; +/// \brief Key for Kernel::Arg::Metadata::mTypeName. +constexpr char TypeName[] = "TypeName"; /// \brief Key for Kernel::Arg::Metadata::mSize. constexpr char Size[] = "Size"; /// \brief Key for Kernel::Arg::Metadata::mAlign. @@ -160,26 +165,28 @@ constexpr char ValueKind[] = "ValueKind"; constexpr char ValueType[] = "ValueType"; /// \brief Key for Kernel::Arg::Metadata::mPointeeAlign. constexpr char PointeeAlign[] = "PointeeAlign"; -/// \brief Key for Kernel::Arg::Metadata::mAccQual. -constexpr char AccQual[] = "AccQual"; /// \brief Key for Kernel::Arg::Metadata::mAddrSpaceQual. constexpr char AddrSpaceQual[] = "AddrSpaceQual"; +/// \brief Key for Kernel::Arg::Metadata::mAccQual. +constexpr char AccQual[] = "AccQual"; +/// \brief Key for Kernel::Arg::Metadata::mActualAccQual. +constexpr char ActualAccQual[] = "ActualAccQual"; /// \brief Key for Kernel::Arg::Metadata::mIsConst. constexpr char IsConst[] = "IsConst"; -/// \brief Key for Kernel::Arg::Metadata::mIsPipe. -constexpr char IsPipe[] = "IsPipe"; /// \brief Key for Kernel::Arg::Metadata::mIsRestrict. constexpr char IsRestrict[] = "IsRestrict"; /// \brief Key for Kernel::Arg::Metadata::mIsVolatile. constexpr char IsVolatile[] = "IsVolatile"; -/// \brief Key for Kernel::Arg::Metadata::mName. -constexpr char Name[] = "Name"; -/// \brief Key for Kernel::Arg::Metadata::mTypeName. -constexpr char TypeName[] = "TypeName"; +/// \brief Key for Kernel::Arg::Metadata::mIsPipe. +constexpr char IsPipe[] = "IsPipe"; } // end namespace Key /// \brief In-memory representation of kernel argument metadata. struct Metadata final { + /// \brief Name. Optional. + std::string mName = std::string(); + /// \brief Type name. Optional. + std::string mTypeName = std::string(); /// \brief Size in bytes. Required. uint32_t mSize = 0; /// \brief Alignment in bytes. Required. @@ -190,22 +197,20 @@ struct Metadata final { ValueType mValueType = ValueType::Unknown; /// \brief Pointee alignment in bytes. Optional. uint32_t mPointeeAlign = 0; - /// \brief Access qualifier. Optional. - AccessQualifier mAccQual = AccessQualifier::Unknown; /// \brief Address space qualifier. Optional. AddressSpaceQualifier mAddrSpaceQual = AddressSpaceQualifier::Unknown; + /// \brief Access qualifier. Optional. + AccessQualifier mAccQual = AccessQualifier::Unknown; + /// \brief Actual access qualifier. Optional. + AccessQualifier mActualAccQual = AccessQualifier::Unknown; /// \brief True if 'const' qualifier is specified. Optional. bool mIsConst = false; - /// \brief True if 'pipe' qualifier is specified. Optional. - bool mIsPipe = false; /// \brief True if 'restrict' qualifier is specified. Optional. bool mIsRestrict = false; /// \brief True if 'volatile' qualifier is specified. Optional. bool mIsVolatile = false; - /// \brief Name. Optional. - std::string mName = std::string(); - /// \brief Type name. Optional. - std::string mTypeName = std::string(); + /// \brief True if 'pipe' qualifier is specified. Optional. + bool mIsPipe = false; /// \brief Default constructor. Metadata() = default; @@ -221,51 +226,63 @@ namespace CodeProps { namespace Key { /// \brief Key for Kernel::CodeProps::Metadata::mKernargSegmentSize. constexpr char KernargSegmentSize[] = "KernargSegmentSize"; -/// \brief Key for Kernel::CodeProps::Metadata::mWorkgroupGroupSegmentSize. -constexpr char WorkgroupGroupSegmentSize[] = "WorkgroupGroupSegmentSize"; -/// \brief Key for Kernel::CodeProps::Metadata::mWorkitemPrivateSegmentSize. -constexpr char WorkitemPrivateSegmentSize[] = "WorkitemPrivateSegmentSize"; -/// \brief Key for Kernel::CodeProps::Metadata::mWavefrontNumSGPRs. -constexpr char WavefrontNumSGPRs[] = "WavefrontNumSGPRs"; -/// \brief Key for Kernel::CodeProps::Metadata::mWorkitemNumVGPRs. -constexpr char WorkitemNumVGPRs[] = "WorkitemNumVGPRs"; +/// \brief Key for Kernel::CodeProps::Metadata::mGroupSegmentFixedSize. +constexpr char GroupSegmentFixedSize[] = "GroupSegmentFixedSize"; +/// \brief Key for Kernel::CodeProps::Metadata::mPrivateSegmentFixedSize. +constexpr char PrivateSegmentFixedSize[] = "PrivateSegmentFixedSize"; /// \brief Key for Kernel::CodeProps::Metadata::mKernargSegmentAlign. constexpr char KernargSegmentAlign[] = "KernargSegmentAlign"; -/// \brief Key for Kernel::CodeProps::Metadata::mGroupSegmentAlign. -constexpr char GroupSegmentAlign[] = "GroupSegmentAlign"; -/// \brief Key for Kernel::CodeProps::Metadata::mPrivateSegmentAlign. -constexpr char PrivateSegmentAlign[] = "PrivateSegmentAlign"; /// \brief Key for Kernel::CodeProps::Metadata::mWavefrontSize. constexpr char WavefrontSize[] = "WavefrontSize"; +/// \brief Key for Kernel::CodeProps::Metadata::mNumSGPRs. +constexpr char NumSGPRs[] = "NumSGPRs"; +/// \brief Key for Kernel::CodeProps::Metadata::mNumVGPRs. +constexpr char NumVGPRs[] = "NumVGPRs"; +/// \brief Key for Kernel::CodeProps::Metadata::mMaxFlatWorkGroupSize. +constexpr char MaxFlatWorkGroupSize[] = "MaxFlatWorkGroupSize"; +/// \brief Key for Kernel::CodeProps::Metadata::mIsDynamicCallStack. +constexpr char IsDynamicCallStack[] = "IsDynamicCallStack"; +/// \brief Key for Kernel::CodeProps::Metadata::mIsXNACKEnabled. +constexpr char IsXNACKEnabled[] = "IsXNACKEnabled"; +/// \brief Key for Kernel::CodeProps::Metadata::mNumSpilledSGPRs. +constexpr char NumSpilledSGPRs[] = "NumSpilledSGPRs"; +/// \brief Key for Kernel::CodeProps::Metadata::mNumSpilledVGPRs. +constexpr char NumSpilledVGPRs[] = "NumSpilledVGPRs"; } // end namespace Key /// \brief In-memory representation of kernel code properties metadata. struct Metadata final { /// \brief Size in bytes of the kernarg segment memory. Kernarg segment memory - /// holds the values of the arguments to the kernel. Optional. + /// holds the values of the arguments to the kernel. Required. uint64_t mKernargSegmentSize = 0; /// \brief Size in bytes of the group segment memory required by a workgroup. /// This value does not include any dynamically allocated group segment memory - /// that may be added when the kernel is dispatched. Optional. - uint32_t mWorkgroupGroupSegmentSize = 0; + /// that may be added when the kernel is dispatched. Required. + uint32_t mGroupSegmentFixedSize = 0; /// \brief Size in bytes of the private segment memory required by a workitem. - /// Private segment memory includes arg, spill and private segments. Optional. - uint32_t mWorkitemPrivateSegmentSize = 0; + /// Private segment memory includes arg, spill and private segments. Required. + uint32_t mPrivateSegmentFixedSize = 0; + /// \brief Maximum byte alignment of variables used by the kernel in the + /// kernarg memory segment. Required. + uint32_t mKernargSegmentAlign = 0; + /// \brief Wavefront size. Required. + uint32_t mWavefrontSize = 0; /// \brief Total number of SGPRs used by a wavefront. Optional. - uint16_t mWavefrontNumSGPRs = 0; + uint16_t mNumSGPRs = 0; /// \brief Total number of VGPRs used by a workitem. Optional. - uint16_t mWorkitemNumVGPRs = 0; - /// \brief Maximum byte alignment of variables used by the kernel in the - /// kernarg memory segment. Expressed as a power of two. Optional. - uint8_t mKernargSegmentAlign = 0; - /// \brief Maximum byte alignment of variables used by the kernel in the - /// group memory segment. Expressed as a power of two. Optional. - uint8_t mGroupSegmentAlign = 0; - /// \brief Maximum byte alignment of variables used by the kernel in the - /// private memory segment. Expressed as a power of two. Optional. - uint8_t mPrivateSegmentAlign = 0; - /// \brief Wavefront size. Expressed as a power of two. Optional. - uint8_t mWavefrontSize = 0; + uint16_t mNumVGPRs = 0; + /// \brief Maximum flat work-group size supported by the kernel. Optional. + uint32_t mMaxFlatWorkGroupSize = 0; + /// \brief True if the generated machine code is using a dynamically sized + /// call stack. Optional. + bool mIsDynamicCallStack = false; + /// \brief True if the generated machine code is capable of supporting XNACK. + /// Optional. + bool mIsXNACKEnabled = false; + /// \brief Number of SGPRs spilled by a wavefront. Optional. + uint16_t mNumSpilledSGPRs = 0; + /// \brief Number of VGPRs spilled by a workitem. Optional. + uint16_t mNumSpilledVGPRs = 0; /// \brief Default constructor. Metadata() = default; @@ -279,10 +296,7 @@ struct Metadata final { /// \returns True if kernel code properties metadata is not empty, false /// otherwise. bool notEmpty() const { - return mKernargSegmentSize || mWorkgroupGroupSegmentSize || - mWorkitemPrivateSegmentSize || mWavefrontNumSGPRs || - mWorkitemNumVGPRs || mKernargSegmentAlign || mGroupSegmentAlign || - mPrivateSegmentAlign || mWavefrontSize; + return true; } }; @@ -348,6 +362,8 @@ struct Metadata final { namespace Key { /// \brief Key for Kernel::Metadata::mName. constexpr char Name[] = "Name"; +/// \brief Key for Kernel::Metadata::mSymbolName. +constexpr char SymbolName[] = "SymbolName"; /// \brief Key for Kernel::Metadata::mLanguage. constexpr char Language[] = "Language"; /// \brief Key for Kernel::Metadata::mLanguageVersion. @@ -364,8 +380,10 @@ constexpr char DebugProps[] = "DebugProps"; /// \brief In-memory representation of kernel metadata. struct Metadata final { - /// \brief Name. Required. + /// \brief Kernel source name. Required. std::string mName = std::string(); + /// \brief Kernel descriptor name. Required. + std::string mSymbolName = std::string(); /// \brief Language. Optional. std::string mLanguage = std::string(); /// \brief Language version. Optional. @@ -386,37 +404,78 @@ struct Metadata final { } // end namespace Kernel namespace Key { -/// \brief Key for CodeObject::Metadata::mVersion. +/// \brief Key for HSA::Metadata::mVersion. constexpr char Version[] = "Version"; -/// \brief Key for CodeObject::Metadata::mPrintf. +/// \brief Key for HSA::Metadata::mPrintf. constexpr char Printf[] = "Printf"; -/// \brief Key for CodeObject::Metadata::mKernels. +/// \brief Key for HSA::Metadata::mKernels. constexpr char Kernels[] = "Kernels"; } // end namespace Key -/// \brief In-memory representation of code object metadata. +/// \brief In-memory representation of HSA metadata. struct Metadata final { - /// \brief Code object metadata version. Required. + /// \brief HSA metadata version. Required. std::vector<uint32_t> mVersion = std::vector<uint32_t>(); /// \brief Printf metadata. Optional. std::vector<std::string> mPrintf = std::vector<std::string>(); - /// \brief Kernels metadata. Optional. + /// \brief Kernels metadata. Required. std::vector<Kernel::Metadata> mKernels = std::vector<Kernel::Metadata>(); /// \brief Default constructor. Metadata() = default; +}; + +/// \brief Converts \p String to \p HSAMetadata. +std::error_code fromString(std::string String, Metadata &HSAMetadata); - /// \brief Converts \p YamlString to \p CodeObjectMetadata. - static std::error_code fromYamlString(std::string YamlString, - Metadata &CodeObjectMetadata); +/// \brief Converts \p HSAMetadata to \p String. +std::error_code toString(Metadata HSAMetadata, std::string &String); - /// \brief Converts \p CodeObjectMetadata to \p YamlString. - static std::error_code toYamlString(Metadata CodeObjectMetadata, - std::string &YamlString); +} // end namespace HSAMD + +//===----------------------------------------------------------------------===// +// PAL metadata. +//===----------------------------------------------------------------------===// +namespace PALMD { + +/// \brief PAL metadata assembler directive. +constexpr char AssemblerDirective[] = ".amd_amdgpu_pal_metadata"; + +/// \brief PAL metadata keys. +enum Key : uint32_t { + LS_NUM_USED_VGPRS = 0x10000021, + HS_NUM_USED_VGPRS = 0x10000022, + ES_NUM_USED_VGPRS = 0x10000023, + GS_NUM_USED_VGPRS = 0x10000024, + VS_NUM_USED_VGPRS = 0x10000025, + PS_NUM_USED_VGPRS = 0x10000026, + CS_NUM_USED_VGPRS = 0x10000027, + + LS_NUM_USED_SGPRS = 0x10000028, + HS_NUM_USED_SGPRS = 0x10000029, + ES_NUM_USED_SGPRS = 0x1000002a, + GS_NUM_USED_SGPRS = 0x1000002b, + VS_NUM_USED_SGPRS = 0x1000002c, + PS_NUM_USED_SGPRS = 0x1000002d, + CS_NUM_USED_SGPRS = 0x1000002e, + + LS_SCRATCH_SIZE = 0x10000044, + HS_SCRATCH_SIZE = 0x10000045, + ES_SCRATCH_SIZE = 0x10000046, + GS_SCRATCH_SIZE = 0x10000047, + VS_SCRATCH_SIZE = 0x10000048, + PS_SCRATCH_SIZE = 0x10000049, + CS_SCRATCH_SIZE = 0x1000004a }; -} // end namespace CodeObject +/// \brief PAL metadata represented as a vector. +typedef std::vector<uint32_t> Metadata; + +/// \brief Converts \p PALMetadata to \p String. +std::error_code toString(const Metadata &PALMetadata, std::string &String); + +} // end namespace PALMD } // end namespace AMDGPU } // end namespace llvm -#endif // LLVM_SUPPORT_AMDGPUCODEOBJECTMETADATA_H +#endif // LLVM_SUPPORT_AMDGPUMETADATA_H diff --git a/contrib/llvm/include/llvm/Support/ARMTargetParser.def b/contrib/llvm/include/llvm/Support/ARMTargetParser.def index 65cb2715a6a5..6c8eff1a8f84 100644 --- a/contrib/llvm/include/llvm/Support/ARMTargetParser.def +++ b/contrib/llvm/include/llvm/Support/ARMTargetParser.def @@ -16,105 +16,109 @@ #ifndef ARM_FPU #define ARM_FPU(NAME, KIND, VERSION, NEON_SUPPORT, RESTRICTION) #endif -ARM_FPU("invalid", FK_INVALID, FV_NONE, NS_None, FR_None) -ARM_FPU("none", FK_NONE, FV_NONE, NS_None, FR_None) -ARM_FPU("vfp", FK_VFP, FV_VFPV2, NS_None, FR_None) -ARM_FPU("vfpv2", FK_VFPV2, FV_VFPV2, NS_None, FR_None) -ARM_FPU("vfpv3", FK_VFPV3, FV_VFPV3, NS_None, FR_None) -ARM_FPU("vfpv3-fp16", FK_VFPV3_FP16, FV_VFPV3_FP16, NS_None, FR_None) -ARM_FPU("vfpv3-d16", FK_VFPV3_D16, FV_VFPV3, NS_None, FR_D16) -ARM_FPU("vfpv3-d16-fp16", FK_VFPV3_D16_FP16, FV_VFPV3_FP16, NS_None, FR_D16) -ARM_FPU("vfpv3xd", FK_VFPV3XD, FV_VFPV3, NS_None, FR_SP_D16) -ARM_FPU("vfpv3xd-fp16", FK_VFPV3XD_FP16, FV_VFPV3_FP16, NS_None, FR_SP_D16) -ARM_FPU("vfpv4", FK_VFPV4, FV_VFPV4, NS_None, FR_None) -ARM_FPU("vfpv4-d16", FK_VFPV4_D16, FV_VFPV4, NS_None, FR_D16) -ARM_FPU("fpv4-sp-d16", FK_FPV4_SP_D16, FV_VFPV4, NS_None, FR_SP_D16) -ARM_FPU("fpv5-d16", FK_FPV5_D16, FV_VFPV5, NS_None, FR_D16) -ARM_FPU("fpv5-sp-d16", FK_FPV5_SP_D16, FV_VFPV5, NS_None, FR_SP_D16) -ARM_FPU("fp-armv8", FK_FP_ARMV8, FV_VFPV5, NS_None, FR_None) -ARM_FPU("neon", FK_NEON, FV_VFPV3, NS_Neon, FR_None) -ARM_FPU("neon-fp16", FK_NEON_FP16, FV_VFPV3_FP16, NS_Neon, FR_None) -ARM_FPU("neon-vfpv4", FK_NEON_VFPV4, FV_VFPV4, NS_Neon, FR_None) -ARM_FPU("neon-fp-armv8", FK_NEON_FP_ARMV8, FV_VFPV5, NS_Neon, FR_None) -ARM_FPU("crypto-neon-fp-armv8", FK_CRYPTO_NEON_FP_ARMV8, FV_VFPV5, NS_Crypto, - FR_None) -ARM_FPU("softvfp", FK_SOFTVFP, FV_NONE, NS_None, FR_None) +ARM_FPU("invalid", FK_INVALID, FPUVersion::NONE, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("none", FK_NONE, FPUVersion::NONE, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfp", FK_VFP, FPUVersion::VFPV2, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfpv2", FK_VFPV2, FPUVersion::VFPV2, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfpv3", FK_VFPV3, FPUVersion::VFPV3, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfpv3-fp16", FK_VFPV3_FP16, FPUVersion::VFPV3_FP16, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfpv3-d16", FK_VFPV3_D16, FPUVersion::VFPV3, NeonSupportLevel::None, FPURestriction::D16) +ARM_FPU("vfpv3-d16-fp16", FK_VFPV3_D16_FP16, FPUVersion::VFPV3_FP16, NeonSupportLevel::None, FPURestriction::D16) +ARM_FPU("vfpv3xd", FK_VFPV3XD, FPUVersion::VFPV3, NeonSupportLevel::None, FPURestriction::SP_D16) +ARM_FPU("vfpv3xd-fp16", FK_VFPV3XD_FP16, FPUVersion::VFPV3_FP16, NeonSupportLevel::None, FPURestriction::SP_D16) +ARM_FPU("vfpv4", FK_VFPV4, FPUVersion::VFPV4, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("vfpv4-d16", FK_VFPV4_D16, FPUVersion::VFPV4, NeonSupportLevel::None, FPURestriction::D16) +ARM_FPU("fpv4-sp-d16", FK_FPV4_SP_D16, FPUVersion::VFPV4, NeonSupportLevel::None, FPURestriction::SP_D16) +ARM_FPU("fpv5-d16", FK_FPV5_D16, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::D16) +ARM_FPU("fpv5-sp-d16", FK_FPV5_SP_D16, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::SP_D16) +ARM_FPU("fp-armv8", FK_FP_ARMV8, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("neon", FK_NEON, FPUVersion::VFPV3, NeonSupportLevel::Neon, FPURestriction::None) +ARM_FPU("neon-fp16", FK_NEON_FP16, FPUVersion::VFPV3_FP16, NeonSupportLevel::Neon, FPURestriction::None) +ARM_FPU("neon-vfpv4", FK_NEON_VFPV4, FPUVersion::VFPV4, NeonSupportLevel::Neon, FPURestriction::None) +ARM_FPU("neon-fp-armv8", FK_NEON_FP_ARMV8, FPUVersion::VFPV5, NeonSupportLevel::Neon, FPURestriction::None) +ARM_FPU("crypto-neon-fp-armv8", FK_CRYPTO_NEON_FP_ARMV8, FPUVersion::VFPV5, NeonSupportLevel::Crypto, + FPURestriction::None) +ARM_FPU("softvfp", FK_SOFTVFP, FPUVersion::NONE, NeonSupportLevel::None, FPURestriction::None) #undef ARM_FPU #ifndef ARM_ARCH #define ARM_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) #endif -ARM_ARCH("invalid", AK_INVALID, nullptr, nullptr, +ARM_ARCH("invalid", INVALID, "", "", ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv2", AK_ARMV2, "2", "v2", ARMBuildAttrs::CPUArch::Pre_v4, +ARM_ARCH("armv2", ARMV2, "2", "v2", ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv2a", AK_ARMV2A, "2A", "v2a", ARMBuildAttrs::CPUArch::Pre_v4, +ARM_ARCH("armv2a", ARMV2A, "2A", "v2a", ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv3", AK_ARMV3, "3", "v3", ARMBuildAttrs::CPUArch::Pre_v4, +ARM_ARCH("armv3", ARMV3, "3", "v3", ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv3m", AK_ARMV3M, "3M", "v3m", ARMBuildAttrs::CPUArch::Pre_v4, +ARM_ARCH("armv3m", ARMV3M, "3M", "v3m", ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv4", AK_ARMV4, "4", "v4", ARMBuildAttrs::CPUArch::v4, +ARM_ARCH("armv4", ARMV4, "4", "v4", ARMBuildAttrs::CPUArch::v4, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv4t", AK_ARMV4T, "4T", "v4t", ARMBuildAttrs::CPUArch::v4T, +ARM_ARCH("armv4t", ARMV4T, "4T", "v4t", ARMBuildAttrs::CPUArch::v4T, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv5t", AK_ARMV5T, "5T", "v5", ARMBuildAttrs::CPUArch::v5T, +ARM_ARCH("armv5t", ARMV5T, "5T", "v5", ARMBuildAttrs::CPUArch::v5T, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv5te", AK_ARMV5TE, "5TE", "v5e", ARMBuildAttrs::CPUArch::v5TE, +ARM_ARCH("armv5te", ARMV5TE, "5TE", "v5e", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, ARM::AEK_DSP) -ARM_ARCH("armv5tej", AK_ARMV5TEJ, "5TEJ", "v5e", ARMBuildAttrs::CPUArch::v5TEJ, +ARM_ARCH("armv5tej", ARMV5TEJ, "5TEJ", "v5e", ARMBuildAttrs::CPUArch::v5TEJ, FK_NONE, ARM::AEK_DSP) -ARM_ARCH("armv6", AK_ARMV6, "6", "v6", ARMBuildAttrs::CPUArch::v6, +ARM_ARCH("armv6", ARMV6, "6", "v6", ARMBuildAttrs::CPUArch::v6, FK_VFPV2, ARM::AEK_DSP) -ARM_ARCH("armv6k", AK_ARMV6K, "6K", "v6k", ARMBuildAttrs::CPUArch::v6K, +ARM_ARCH("armv6k", ARMV6K, "6K", "v6k", ARMBuildAttrs::CPUArch::v6K, FK_VFPV2, ARM::AEK_DSP) -ARM_ARCH("armv6t2", AK_ARMV6T2, "6T2", "v6t2", ARMBuildAttrs::CPUArch::v6T2, +ARM_ARCH("armv6t2", ARMV6T2, "6T2", "v6t2", ARMBuildAttrs::CPUArch::v6T2, FK_NONE, ARM::AEK_DSP) -ARM_ARCH("armv6kz", AK_ARMV6KZ, "6KZ", "v6kz", ARMBuildAttrs::CPUArch::v6KZ, +ARM_ARCH("armv6kz", ARMV6KZ, "6KZ", "v6kz", ARMBuildAttrs::CPUArch::v6KZ, FK_VFPV2, (ARM::AEK_SEC | ARM::AEK_DSP)) -ARM_ARCH("armv6-m", AK_ARMV6M, "6-M", "v6m", ARMBuildAttrs::CPUArch::v6_M, +ARM_ARCH("armv6-m", ARMV6M, "6-M", "v6m", ARMBuildAttrs::CPUArch::v6_M, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv7-a", AK_ARMV7A, "7-A", "v7", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7-a", ARMV7A, "7-A", "v7", ARMBuildAttrs::CPUArch::v7, FK_NEON, ARM::AEK_DSP) -ARM_ARCH("armv7ve", AK_ARMV7VE, "7VE", "v7ve", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7ve", ARMV7VE, "7VE", "v7ve", ARMBuildAttrs::CPUArch::v7, FK_NEON, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP)) -ARM_ARCH("armv7-r", AK_ARMV7R, "7-R", "v7r", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7-r", ARMV7R, "7-R", "v7r", ARMBuildAttrs::CPUArch::v7, FK_NONE, (ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP)) -ARM_ARCH("armv7-m", AK_ARMV7M, "7-M", "v7m", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7-m", ARMV7M, "7-M", "v7m", ARMBuildAttrs::CPUArch::v7, FK_NONE, ARM::AEK_HWDIVTHUMB) -ARM_ARCH("armv7e-m", AK_ARMV7EM, "7E-M", "v7em", ARMBuildAttrs::CPUArch::v7E_M, +ARM_ARCH("armv7e-m", ARMV7EM, "7E-M", "v7em", ARMBuildAttrs::CPUArch::v7E_M, FK_NONE, (ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP)) -ARM_ARCH("armv8-a", AK_ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8_A, +ARM_ARCH("armv8-a", ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC)) -ARM_ARCH("armv8.1-a", AK_ARMV8_1A, "8.1-A", "v8.1a", +ARM_ARCH("armv8.1-a", ARMV8_1A, "8.1-A", "v8.1a", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC)) -ARM_ARCH("armv8.2-a", AK_ARMV8_2A, "8.2-A", "v8.2a", +ARM_ARCH("armv8.2-a", ARMV8_2A, "8.2-A", "v8.2a", ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS)) -ARM_ARCH("armv8-r", AK_ARMV8R, "8-R", "v8r", ARMBuildAttrs::CPUArch::v8_R, +ARM_ARCH("armv8.3-a", ARMV8_3A, "8.3-A", "v8.3a", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS)) +ARM_ARCH("armv8-r", ARMV8R, "8-R", "v8r", ARMBuildAttrs::CPUArch::v8_R, FK_NEON_FP_ARMV8, (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC)) -ARM_ARCH("armv8-m.base", AK_ARMV8MBaseline, "8-M.Baseline", "v8m.base", +ARM_ARCH("armv8-m.base", ARMV8MBaseline, "8-M.Baseline", "v8m.base", ARMBuildAttrs::CPUArch::v8_M_Base, FK_NONE, ARM::AEK_HWDIVTHUMB) -ARM_ARCH("armv8-m.main", AK_ARMV8MMainline, "8-M.Mainline", "v8m.main", +ARM_ARCH("armv8-m.main", ARMV8MMainline, "8-M.Mainline", "v8m.main", ARMBuildAttrs::CPUArch::v8_M_Main, FK_FPV5_D16, ARM::AEK_HWDIVTHUMB) // Non-standard Arch names. -ARM_ARCH("iwmmxt", AK_IWMMXT, "iwmmxt", "", ARMBuildAttrs::CPUArch::v5TE, +ARM_ARCH("iwmmxt", IWMMXT, "iwmmxt", "", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("iwmmxt2", AK_IWMMXT2, "iwmmxt2", "", ARMBuildAttrs::CPUArch::v5TE, +ARM_ARCH("iwmmxt2", IWMMXT2, "iwmmxt2", "", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("xscale", AK_XSCALE, "xscale", "v5e", ARMBuildAttrs::CPUArch::v5TE, +ARM_ARCH("xscale", XSCALE, "xscale", "v5e", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, ARM::AEK_NONE) -ARM_ARCH("armv7s", AK_ARMV7S, "7-S", "v7s", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7s", ARMV7S, "7-S", "v7s", ARMBuildAttrs::CPUArch::v7, FK_NEON_VFPV4, ARM::AEK_DSP) -ARM_ARCH("armv7k", AK_ARMV7K, "7-K", "v7k", ARMBuildAttrs::CPUArch::v7, +ARM_ARCH("armv7k", ARMV7K, "7-K", "v7k", ARMBuildAttrs::CPUArch::v7, FK_NONE, ARM::AEK_DSP) #undef ARM_ARCH @@ -126,6 +130,7 @@ ARM_ARCH_EXT_NAME("invalid", ARM::AEK_INVALID, nullptr, nullptr) ARM_ARCH_EXT_NAME("none", ARM::AEK_NONE, nullptr, nullptr) ARM_ARCH_EXT_NAME("crc", ARM::AEK_CRC, "+crc", "-crc") ARM_ARCH_EXT_NAME("crypto", ARM::AEK_CRYPTO, "+crypto","-crypto") +ARM_ARCH_EXT_NAME("dotprod", ARM::AEK_DOTPROD, "+dotprod","-dotprod") ARM_ARCH_EXT_NAME("dsp", ARM::AEK_DSP, "+dsp", "-dsp") ARM_ARCH_EXT_NAME("fp", ARM::AEK_FP, nullptr, nullptr) ARM_ARCH_EXT_NAME("idiv", (ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB), nullptr, nullptr) @@ -155,101 +160,105 @@ ARM_HW_DIV_NAME("arm,thumb", (ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) #ifndef ARM_CPU_NAME #define ARM_CPU_NAME(NAME, ID, DEFAULT_FPU, IS_DEFAULT, DEFAULT_EXT) #endif -ARM_CPU_NAME("arm2", AK_ARMV2, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm3", AK_ARMV2A, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm6", AK_ARMV3, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm7m", AK_ARMV3M, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm8", AK_ARMV4, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm810", AK_ARMV4, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("strongarm", AK_ARMV4, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("strongarm110", AK_ARMV4, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("strongarm1100", AK_ARMV4, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("strongarm1110", AK_ARMV4, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm7tdmi", AK_ARMV4T, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm7tdmi-s", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm710t", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm720t", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm9", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm9tdmi", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm920", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm920t", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm922t", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm9312", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm940t", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("ep9312", AK_ARMV4T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm10tdmi", AK_ARMV5T, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1020t", AK_ARMV5T, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm9e", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm946e-s", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm966e-s", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm968e-s", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm10e", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm1020e", AK_ARMV5TE, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm1022e", AK_ARMV5TE, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm926ej-s", AK_ARMV5TEJ, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1136j-s", AK_ARMV6, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm1136jf-s", AK_ARMV6, FK_VFPV2, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1136jz-s", AK_ARMV6, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm1176j-s", AK_ARMV6K, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1176jz-s", AK_ARMV6KZ, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("mpcore", AK_ARMV6K, FK_VFPV2, false, ARM::AEK_NONE) -ARM_CPU_NAME("mpcorenovfp", AK_ARMV6K, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("arm1176jzf-s", AK_ARMV6KZ, FK_VFPV2, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1156t2-s", AK_ARMV6T2, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("arm1156t2f-s", AK_ARMV6T2, FK_VFPV2, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m0", AK_ARMV6M, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m0plus", AK_ARMV6M, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m1", AK_ARMV6M, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("sc000", AK_ARMV6M, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-a5", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("arm2", ARMV2, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm3", ARMV2A, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm6", ARMV3, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm7m", ARMV3M, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm8", ARMV4, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm810", ARMV4, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("strongarm", ARMV4, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("strongarm110", ARMV4, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("strongarm1100", ARMV4, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("strongarm1110", ARMV4, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm7tdmi", ARMV4T, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm7tdmi-s", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm710t", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm720t", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm9", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm9tdmi", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm920", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm920t", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm922t", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm9312", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm940t", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("ep9312", ARMV4T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm10tdmi", ARMV5T, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1020t", ARMV5T, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm9e", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm946e-s", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm966e-s", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm968e-s", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm10e", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm1020e", ARMV5TE, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm1022e", ARMV5TE, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm926ej-s", ARMV5TEJ, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1136j-s", ARMV6, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm1136jf-s", ARMV6, FK_VFPV2, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1136jz-s", ARMV6, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm1176j-s", ARMV6K, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1176jz-s", ARMV6KZ, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("mpcore", ARMV6K, FK_VFPV2, false, ARM::AEK_NONE) +ARM_CPU_NAME("mpcorenovfp", ARMV6K, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("arm1176jzf-s", ARMV6KZ, FK_VFPV2, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1156t2-s", ARMV6T2, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("arm1156t2f-s", ARMV6T2, FK_VFPV2, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m0", ARMV6M, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m0plus", ARMV6M, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m1", ARMV6M, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("sc000", ARMV6M, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-a5", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_SEC | ARM::AEK_MP)) -ARM_CPU_NAME("cortex-a7", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("cortex-a7", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) -ARM_CPU_NAME("cortex-a8", AK_ARMV7A, FK_NEON, false, ARM::AEK_SEC) -ARM_CPU_NAME("cortex-a9", AK_ARMV7A, FK_NEON_FP16, false, (ARM::AEK_SEC | ARM::AEK_MP)) -ARM_CPU_NAME("cortex-a12", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("cortex-a8", ARMV7A, FK_NEON, false, ARM::AEK_SEC) +ARM_CPU_NAME("cortex-a9", ARMV7A, FK_NEON_FP16, false, (ARM::AEK_SEC | ARM::AEK_MP)) +ARM_CPU_NAME("cortex-a12", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) -ARM_CPU_NAME("cortex-a15", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("cortex-a15", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) -ARM_CPU_NAME("cortex-a17", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("cortex-a17", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) -ARM_CPU_NAME("krait", AK_ARMV7A, FK_NEON_VFPV4, false, +ARM_CPU_NAME("krait", ARMV7A, FK_NEON_VFPV4, false, (ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) -ARM_CPU_NAME("cortex-r4", AK_ARMV7R, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-r4f", AK_ARMV7R, FK_VFPV3_D16, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-r5", AK_ARMV7R, FK_VFPV3_D16, false, +ARM_CPU_NAME("cortex-r4", ARMV7R, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-r4f", ARMV7R, FK_VFPV3_D16, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-r5", ARMV7R, FK_VFPV3_D16, false, (ARM::AEK_MP | ARM::AEK_HWDIVARM)) -ARM_CPU_NAME("cortex-r7", AK_ARMV7R, FK_VFPV3_D16_FP16, false, +ARM_CPU_NAME("cortex-r7", ARMV7R, FK_VFPV3_D16_FP16, false, (ARM::AEK_MP | ARM::AEK_HWDIVARM)) -ARM_CPU_NAME("cortex-r8", AK_ARMV7R, FK_VFPV3_D16_FP16, false, +ARM_CPU_NAME("cortex-r8", ARMV7R, FK_VFPV3_D16_FP16, false, (ARM::AEK_MP | ARM::AEK_HWDIVARM)) -ARM_CPU_NAME("cortex-r52", AK_ARMV8R, FK_NEON_FP_ARMV8, true, ARM::AEK_NONE) -ARM_CPU_NAME("sc300", AK_ARMV7M, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m3", AK_ARMV7M, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m4", AK_ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m7", AK_ARMV7EM, FK_FPV5_D16, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m23", AK_ARMV8MBaseline, FK_NONE, false, ARM::AEK_NONE) -ARM_CPU_NAME("cortex-m33", AK_ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP) -ARM_CPU_NAME("cortex-a32", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cortex-a35", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cortex-a53", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cortex-a57", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cortex-a72", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cortex-a73", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("cyclone", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("exynos-m1", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("exynos-m2", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("exynos-m3", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("kryo", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-r52", ARMV8R, FK_NEON_FP_ARMV8, true, ARM::AEK_NONE) +ARM_CPU_NAME("sc300", ARMV7M, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m3", ARMV7M, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m7", ARMV7EM, FK_FPV5_D16, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m23", ARMV8MBaseline, FK_NONE, false, ARM::AEK_NONE) +ARM_CPU_NAME("cortex-m33", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP) +ARM_CPU_NAME("cortex-a32", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a55", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("cortex-a75", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) +ARM_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) // Non-standard Arch names. -ARM_CPU_NAME("iwmmxt", AK_IWMMXT, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("xscale", AK_XSCALE, FK_NONE, true, ARM::AEK_NONE) -ARM_CPU_NAME("swift", AK_ARMV7S, FK_NEON_VFPV4, true, +ARM_CPU_NAME("iwmmxt", IWMMXT, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("xscale", XSCALE, FK_NONE, true, ARM::AEK_NONE) +ARM_CPU_NAME("swift", ARMV7S, FK_NEON_VFPV4, true, (ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB)) // Invalid CPU -ARM_CPU_NAME("invalid", AK_INVALID, FK_INVALID, true, ARM::AEK_INVALID) +ARM_CPU_NAME("invalid", INVALID, FK_INVALID, true, ARM::AEK_INVALID) #undef ARM_CPU_NAME diff --git a/contrib/llvm/include/llvm/Support/Allocator.h b/contrib/llvm/include/llvm/Support/Allocator.h index a5e662f4c588..a94aa8fb1f2a 100644 --- a/contrib/llvm/include/llvm/Support/Allocator.h +++ b/contrib/llvm/include/llvm/Support/Allocator.h @@ -269,6 +269,9 @@ public: // Pull in base class overloads. using AllocatorBase<BumpPtrAllocatorImpl>::Allocate; + // Bump pointer allocators are expected to never free their storage; and + // clients expect pointers to remain valid for non-dereferencing uses even + // after deallocation. void Deallocate(const void *Ptr, size_t Size) { __asan_poison_memory_region(Ptr, Size); } diff --git a/contrib/llvm/include/llvm/Support/AtomicOrdering.h b/contrib/llvm/include/llvm/Support/AtomicOrdering.h index 001804248b85..e93b755aa63b 100644 --- a/contrib/llvm/include/llvm/Support/AtomicOrdering.h +++ b/contrib/llvm/include/llvm/Support/AtomicOrdering.h @@ -42,7 +42,7 @@ bool operator>=(AtomicOrderingCABI, AtomicOrderingCABI) = delete; // Validate an integral value which isn't known to fit within the enum's range // is a valid AtomicOrderingCABI. -template <typename Int> static inline bool isValidAtomicOrderingCABI(Int I) { +template <typename Int> inline bool isValidAtomicOrderingCABI(Int I) { return (Int)AtomicOrderingCABI::relaxed <= I && I <= (Int)AtomicOrderingCABI::seq_cst; } @@ -72,13 +72,13 @@ bool operator>=(AtomicOrdering, AtomicOrdering) = delete; // Validate an integral value which isn't known to fit within the enum's range // is a valid AtomicOrdering. -template <typename Int> static inline bool isValidAtomicOrdering(Int I) { +template <typename Int> inline bool isValidAtomicOrdering(Int I) { return static_cast<Int>(AtomicOrdering::NotAtomic) <= I && I <= static_cast<Int>(AtomicOrdering::SequentiallyConsistent); } /// String used by LLVM IR to represent atomic ordering. -static inline const char *toIRString(AtomicOrdering ao) { +inline const char *toIRString(AtomicOrdering ao) { static const char *names[8] = {"not_atomic", "unordered", "monotonic", "consume", "acquire", "release", "acq_rel", "seq_cst"}; @@ -87,7 +87,7 @@ static inline const char *toIRString(AtomicOrdering ao) { /// Returns true if ao is stronger than other as defined by the AtomicOrdering /// lattice, which is based on C++'s definition. -static inline bool isStrongerThan(AtomicOrdering ao, AtomicOrdering other) { +inline bool isStrongerThan(AtomicOrdering ao, AtomicOrdering other) { static const bool lookup[8][8] = { // NA UN RX CO AC RE AR SC /* NotAtomic */ {false, false, false, false, false, false, false, false}, @@ -102,8 +102,7 @@ static inline bool isStrongerThan(AtomicOrdering ao, AtomicOrdering other) { return lookup[static_cast<size_t>(ao)][static_cast<size_t>(other)]; } -static inline bool isAtLeastOrStrongerThan(AtomicOrdering ao, - AtomicOrdering other) { +inline bool isAtLeastOrStrongerThan(AtomicOrdering ao, AtomicOrdering other) { static const bool lookup[8][8] = { // NA UN RX CO AC RE AR SC /* NotAtomic */ { true, false, false, false, false, false, false, false}, @@ -118,23 +117,23 @@ static inline bool isAtLeastOrStrongerThan(AtomicOrdering ao, return lookup[static_cast<size_t>(ao)][static_cast<size_t>(other)]; } -static inline bool isStrongerThanUnordered(AtomicOrdering ao) { +inline bool isStrongerThanUnordered(AtomicOrdering ao) { return isStrongerThan(ao, AtomicOrdering::Unordered); } -static inline bool isStrongerThanMonotonic(AtomicOrdering ao) { +inline bool isStrongerThanMonotonic(AtomicOrdering ao) { return isStrongerThan(ao, AtomicOrdering::Monotonic); } -static inline bool isAcquireOrStronger(AtomicOrdering ao) { +inline bool isAcquireOrStronger(AtomicOrdering ao) { return isAtLeastOrStrongerThan(ao, AtomicOrdering::Acquire); } -static inline bool isReleaseOrStronger(AtomicOrdering ao) { +inline bool isReleaseOrStronger(AtomicOrdering ao) { return isAtLeastOrStrongerThan(ao, AtomicOrdering::Release); } -static inline AtomicOrderingCABI toCABI(AtomicOrdering ao) { +inline AtomicOrderingCABI toCABI(AtomicOrdering ao) { static const AtomicOrderingCABI lookup[8] = { /* NotAtomic */ AtomicOrderingCABI::relaxed, /* Unordered */ AtomicOrderingCABI::relaxed, diff --git a/contrib/llvm/include/llvm/Support/BinaryByteStream.h b/contrib/llvm/include/llvm/Support/BinaryByteStream.h index 694be28e07e1..db1ccba1398b 100644 --- a/contrib/llvm/include/llvm/Support/BinaryByteStream.h +++ b/contrib/llvm/include/llvm/Support/BinaryByteStream.h @@ -41,7 +41,7 @@ public: Error readBytes(uint32_t Offset, uint32_t Size, ArrayRef<uint8_t> &Buffer) override { - if (auto EC = checkOffset(Offset, Size)) + if (auto EC = checkOffsetForRead(Offset, Size)) return EC; Buffer = Data.slice(Offset, Size); return Error::success(); @@ -49,7 +49,7 @@ public: Error readLongestContiguousChunk(uint32_t Offset, ArrayRef<uint8_t> &Buffer) override { - if (auto EC = checkOffset(Offset, 1)) + if (auto EC = checkOffsetForRead(Offset, 1)) return EC; Buffer = Data.slice(Offset); return Error::success(); @@ -114,7 +114,7 @@ public: if (Buffer.empty()) return Error::success(); - if (auto EC = checkOffset(Offset, Buffer.size())) + if (auto EC = checkOffsetForWrite(Offset, Buffer.size())) return EC; uint8_t *DataPtr = const_cast<uint8_t *>(Data.data()); @@ -131,6 +131,76 @@ private: BinaryByteStream ImmutableStream; }; +/// \brief An implementation of WritableBinaryStream which can write at its end +/// causing the underlying data to grow. This class owns the underlying data. +class AppendingBinaryByteStream : public WritableBinaryStream { + std::vector<uint8_t> Data; + llvm::support::endianness Endian = llvm::support::little; + +public: + AppendingBinaryByteStream() = default; + AppendingBinaryByteStream(llvm::support::endianness Endian) + : Endian(Endian) {} + + void clear() { Data.clear(); } + + llvm::support::endianness getEndian() const override { return Endian; } + + Error readBytes(uint32_t Offset, uint32_t Size, + ArrayRef<uint8_t> &Buffer) override { + if (auto EC = checkOffsetForWrite(Offset, Buffer.size())) + return EC; + + Buffer = makeArrayRef(Data).slice(Offset, Size); + return Error::success(); + } + + void insert(uint32_t Offset, ArrayRef<uint8_t> Bytes) { + Data.insert(Data.begin() + Offset, Bytes.begin(), Bytes.end()); + } + + Error readLongestContiguousChunk(uint32_t Offset, + ArrayRef<uint8_t> &Buffer) override { + if (auto EC = checkOffsetForWrite(Offset, 1)) + return EC; + + Buffer = makeArrayRef(Data).slice(Offset); + return Error::success(); + } + + uint32_t getLength() override { return Data.size(); } + + Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> Buffer) override { + if (Buffer.empty()) + return Error::success(); + + // This is well-defined for any case except where offset is strictly + // greater than the current length. If offset is equal to the current + // length, we can still grow. If offset is beyond the current length, we + // would have to decide how to deal with the intermediate uninitialized + // bytes. So we punt on that case for simplicity and just say it's an + // error. + if (Offset > getLength()) + return make_error<BinaryStreamError>(stream_error_code::invalid_offset); + + uint32_t RequiredSize = Offset + Buffer.size(); + if (RequiredSize > Data.size()) + Data.resize(RequiredSize); + + ::memcpy(Data.data() + Offset, Buffer.data(), Buffer.size()); + return Error::success(); + } + + Error commit() override { return Error::success(); } + + /// \brief Return the properties of this stream. + virtual BinaryStreamFlags getFlags() const override { + return BSF_Write | BSF_Append; + } + + MutableArrayRef<uint8_t> data() { return Data; } +}; + /// \brief An implementation of WritableBinaryStream backed by an llvm /// FileOutputBuffer. class FileBufferByteStream : public WritableBinaryStream { diff --git a/contrib/llvm/include/llvm/Support/BinaryItemStream.h b/contrib/llvm/include/llvm/Support/BinaryItemStream.h index fe7e6caeaafb..278723ddf8da 100644 --- a/contrib/llvm/include/llvm/Support/BinaryItemStream.h +++ b/contrib/llvm/include/llvm/Support/BinaryItemStream.h @@ -45,7 +45,7 @@ public: if (!ExpectedIndex) return ExpectedIndex.takeError(); const auto &Item = Items[*ExpectedIndex]; - if (auto EC = checkOffset(Offset, Size)) + if (auto EC = checkOffsetForRead(Offset, Size)) return EC; if (Size > Traits::length(Item)) return make_error<BinaryStreamError>(stream_error_code::stream_too_short); diff --git a/contrib/llvm/include/llvm/Support/BinaryStream.h b/contrib/llvm/include/llvm/Support/BinaryStream.h index a227117e063e..d69a03eccfdb 100644 --- a/contrib/llvm/include/llvm/Support/BinaryStream.h +++ b/contrib/llvm/include/llvm/Support/BinaryStream.h @@ -11,6 +11,7 @@ #define LLVM_SUPPORT_BINARYSTREAM_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/Support/BinaryStreamError.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -18,6 +19,13 @@ namespace llvm { +enum BinaryStreamFlags { + BSF_None = 0, + BSF_Write = 1, // Stream supports writing. + BSF_Append = 2, // Writing can occur at offset == length. + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ BSF_Append) +}; + /// \brief An interface for accessing data in a stream-like format, but which /// discourages copying. Instead of specifying a buffer in which to copy /// data on a read, the API returns an ArrayRef to data owned by the stream's @@ -45,8 +53,11 @@ public: /// \brief Return the number of bytes of data in this stream. virtual uint32_t getLength() = 0; + /// \brief Return the properties of this stream. + virtual BinaryStreamFlags getFlags() const { return BSF_None; } + protected: - Error checkOffset(uint32_t Offset, uint32_t DataSize) { + Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) { if (Offset > getLength()) return make_error<BinaryStreamError>(stream_error_code::invalid_offset); if (getLength() < DataSize + Offset) @@ -71,6 +82,19 @@ public: /// \brief For buffered streams, commits changes to the backing store. virtual Error commit() = 0; + + /// \brief Return the properties of this stream. + BinaryStreamFlags getFlags() const override { return BSF_Write; } + +protected: + Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) { + if (!(getFlags() & BSF_Append)) + return checkOffsetForRead(Offset, DataSize); + + if (Offset > getLength()) + return make_error<BinaryStreamError>(stream_error_code::invalid_offset); + return Error::success(); + } }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Support/BinaryStreamRef.h b/contrib/llvm/include/llvm/Support/BinaryStreamRef.h index 6d5135cb258d..5cf355be6fe9 100644 --- a/contrib/llvm/include/llvm/Support/BinaryStreamRef.h +++ b/contrib/llvm/include/llvm/Support/BinaryStreamRef.h @@ -11,6 +11,7 @@ #define LLVM_SUPPORT_BINARYSTREAMREF_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/BinaryStream.h" #include "llvm/Support/BinaryStreamError.h" #include "llvm/Support/Error.h" @@ -24,47 +25,74 @@ namespace llvm { template <class RefType, class StreamType> class BinaryStreamRefBase { protected: BinaryStreamRefBase() = default; + explicit BinaryStreamRefBase(StreamType &BorrowedImpl) + : BorrowedImpl(&BorrowedImpl), ViewOffset(0) { + if (!(BorrowedImpl.getFlags() & BSF_Append)) + Length = BorrowedImpl.getLength(); + } + BinaryStreamRefBase(std::shared_ptr<StreamType> SharedImpl, uint32_t Offset, - uint32_t Length) + Optional<uint32_t> Length) : SharedImpl(SharedImpl), BorrowedImpl(SharedImpl.get()), ViewOffset(Offset), Length(Length) {} BinaryStreamRefBase(StreamType &BorrowedImpl, uint32_t Offset, - uint32_t Length) + Optional<uint32_t> Length) : BorrowedImpl(&BorrowedImpl), ViewOffset(Offset), Length(Length) {} - BinaryStreamRefBase(const BinaryStreamRefBase &Other) { - SharedImpl = Other.SharedImpl; - BorrowedImpl = Other.BorrowedImpl; - ViewOffset = Other.ViewOffset; - Length = Other.Length; - } + BinaryStreamRefBase(const BinaryStreamRefBase &Other) = default; + BinaryStreamRefBase &operator=(const BinaryStreamRefBase &Other) = default; + + BinaryStreamRefBase &operator=(BinaryStreamRefBase &&Other) = default; + BinaryStreamRefBase(BinaryStreamRefBase &&Other) = default; public: llvm::support::endianness getEndian() const { return BorrowedImpl->getEndian(); } - uint32_t getLength() const { return Length; } + uint32_t getLength() const { + if (Length.hasValue()) + return *Length; - /// Return a new BinaryStreamRef with the first \p N elements removed. + return BorrowedImpl ? (BorrowedImpl->getLength() - ViewOffset) : 0; + } + + /// Return a new BinaryStreamRef with the first \p N elements removed. If + /// this BinaryStreamRef is length-tracking, then the resulting one will be + /// too. RefType drop_front(uint32_t N) const { if (!BorrowedImpl) return RefType(); - N = std::min(N, Length); + N = std::min(N, getLength()); RefType Result(static_cast<const RefType &>(*this)); + if (N == 0) + return Result; + Result.ViewOffset += N; - Result.Length -= N; + if (Result.Length.hasValue()) + *Result.Length -= N; return Result; } - /// Return a new BinaryStreamRef with the first \p N elements removed. + /// Return a new BinaryStreamRef with the last \p N elements removed. If + /// this BinaryStreamRef is length-tracking and \p N is greater than 0, then + /// this BinaryStreamRef will no longer length-track. RefType drop_back(uint32_t N) const { if (!BorrowedImpl) return RefType(); - N = std::min(N, Length); RefType Result(static_cast<const RefType &>(*this)); - Result.Length -= N; + N = std::min(N, getLength()); + + if (N == 0) + return Result; + + // Since we're dropping non-zero bytes from the end, stop length-tracking + // by setting the length of the resulting StreamRef to an explicit value. + if (!Result.Length.hasValue()) + Result.Length = getLength(); + + *Result.Length -= N; return Result; } @@ -105,7 +133,7 @@ public: } protected: - Error checkOffset(uint32_t Offset, uint32_t DataSize) const { + Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) const { if (Offset > getLength()) return make_error<BinaryStreamError>(stream_error_code::invalid_offset); if (getLength() < DataSize + Offset) @@ -116,7 +144,7 @@ protected: std::shared_ptr<StreamType> SharedImpl; StreamType *BorrowedImpl = nullptr; uint32_t ViewOffset = 0; - uint32_t Length = 0; + Optional<uint32_t> Length; }; /// \brief BinaryStreamRef is to BinaryStream what ArrayRef is to an Array. It @@ -131,18 +159,22 @@ class BinaryStreamRef friend BinaryStreamRefBase<BinaryStreamRef, BinaryStream>; friend class WritableBinaryStreamRef; BinaryStreamRef(std::shared_ptr<BinaryStream> Impl, uint32_t ViewOffset, - uint32_t Length) + Optional<uint32_t> Length) : BinaryStreamRefBase(Impl, ViewOffset, Length) {} public: BinaryStreamRef() = default; BinaryStreamRef(BinaryStream &Stream); - BinaryStreamRef(BinaryStream &Stream, uint32_t Offset, uint32_t Length); + BinaryStreamRef(BinaryStream &Stream, uint32_t Offset, + Optional<uint32_t> Length); explicit BinaryStreamRef(ArrayRef<uint8_t> Data, llvm::support::endianness Endian); explicit BinaryStreamRef(StringRef Data, llvm::support::endianness Endian); - BinaryStreamRef(const BinaryStreamRef &Other); + BinaryStreamRef(const BinaryStreamRef &Other) = default; + BinaryStreamRef &operator=(const BinaryStreamRef &Other) = default; + BinaryStreamRef(BinaryStreamRef &&Other) = default; + BinaryStreamRef &operator=(BinaryStreamRef &&Other) = default; // Use BinaryStreamRef.slice() instead. BinaryStreamRef(BinaryStreamRef &S, uint32_t Offset, @@ -193,17 +225,31 @@ class WritableBinaryStreamRef WritableBinaryStream> { friend BinaryStreamRefBase<WritableBinaryStreamRef, WritableBinaryStream>; WritableBinaryStreamRef(std::shared_ptr<WritableBinaryStream> Impl, - uint32_t ViewOffset, uint32_t Length) + uint32_t ViewOffset, Optional<uint32_t> Length) : BinaryStreamRefBase(Impl, ViewOffset, Length) {} + Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) const { + if (!(BorrowedImpl->getFlags() & BSF_Append)) + return checkOffsetForRead(Offset, DataSize); + + if (Offset > getLength()) + return make_error<BinaryStreamError>(stream_error_code::invalid_offset); + return Error::success(); + } + public: WritableBinaryStreamRef() = default; WritableBinaryStreamRef(WritableBinaryStream &Stream); WritableBinaryStreamRef(WritableBinaryStream &Stream, uint32_t Offset, - uint32_t Length); + Optional<uint32_t> Length); explicit WritableBinaryStreamRef(MutableArrayRef<uint8_t> Data, llvm::support::endianness Endian); - WritableBinaryStreamRef(const WritableBinaryStreamRef &Other); + WritableBinaryStreamRef(const WritableBinaryStreamRef &Other) = default; + WritableBinaryStreamRef & + operator=(const WritableBinaryStreamRef &Other) = default; + + WritableBinaryStreamRef(WritableBinaryStreamRef &&Other) = default; + WritableBinaryStreamRef &operator=(WritableBinaryStreamRef &&Other) = default; // Use WritableBinaryStreamRef.slice() instead. WritableBinaryStreamRef(WritableBinaryStreamRef &S, uint32_t Offset, diff --git a/contrib/llvm/include/llvm/Support/CachePruning.h b/contrib/llvm/include/llvm/Support/CachePruning.h index 46e34358573b..c577e9b8b631 100644 --- a/contrib/llvm/include/llvm/Support/CachePruning.h +++ b/contrib/llvm/include/llvm/Support/CachePruning.h @@ -46,6 +46,15 @@ struct CachePruningPolicy { /// of available space on the disk will be reduced to the amount of available /// space. A value of 0 disables the absolute size-based pruning. uint64_t MaxSizeBytes = 0; + + /// The maximum number of files in the cache directory. A value of 0 disables + /// the number of files based pruning. + /// + /// This defaults to 1000000 because with that many files there are + /// diminishing returns on the effectiveness of the cache, and some file + /// systems have a limit on how many files can be contained in a directory + /// (notably ext4, which is limited to around 6000000 files). + uint64_t MaxSizeFiles = 1000000; }; /// Parse the given string as a cache pruning policy. Defaults are taken from a diff --git a/contrib/llvm/include/llvm/Support/Chrono.h b/contrib/llvm/include/llvm/Support/Chrono.h index 6118ed0476ed..994068af3771 100644 --- a/contrib/llvm/include/llvm/Support/Chrono.h +++ b/contrib/llvm/include/llvm/Support/Chrono.h @@ -51,6 +51,20 @@ toTimePoint(std::time_t T) { raw_ostream &operator<<(raw_ostream &OS, sys::TimePoint<> TP); +/// Format provider for TimePoint<> +/// +/// The options string is a strftime format string, with extensions: +/// - %L is millis: 000-999 +/// - %f is micros: 000000-999999 +/// - %N is nanos: 000000000 - 999999999 +/// +/// If no options are given, the default format is "%Y-%m-%d %H:%M:%S.%N". +template <> +struct format_provider<sys::TimePoint<>> { + static void format(const sys::TimePoint<> &TP, llvm::raw_ostream &OS, + StringRef Style); +}; + /// Implementation of format_provider<T> for duration types. /// /// The options string of a duration type has the grammar: diff --git a/contrib/llvm/include/llvm/Support/CodeGen.h b/contrib/llvm/include/llvm/Support/CodeGen.h index 941c112b0dd2..5f9e33129587 100644 --- a/contrib/llvm/include/llvm/Support/CodeGen.h +++ b/contrib/llvm/include/llvm/Support/CodeGen.h @@ -25,7 +25,7 @@ namespace llvm { // Code model types. namespace CodeModel { // Sync changes with CodeGenCWrappers.h. - enum Model { Default, JITDefault, Small, Kernel, Medium, Large }; + enum Model { Small, Kernel, Medium, Large }; } namespace PICLevel { diff --git a/contrib/llvm/include/llvm/Support/CodeGenCWrappers.h b/contrib/llvm/include/llvm/Support/CodeGenCWrappers.h index 6db4433a4350..47971e80cefb 100644 --- a/contrib/llvm/include/llvm/Support/CodeGenCWrappers.h +++ b/contrib/llvm/include/llvm/Support/CodeGenCWrappers.h @@ -17,17 +17,20 @@ #define LLVM_SUPPORT_CODEGENCWRAPPERS_H #include "llvm-c/TargetMachine.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" namespace llvm { -inline CodeModel::Model unwrap(LLVMCodeModel Model) { +inline Optional<CodeModel::Model> unwrap(LLVMCodeModel Model, bool &JIT) { + JIT = false; switch (Model) { - case LLVMCodeModelDefault: - return CodeModel::Default; case LLVMCodeModelJITDefault: - return CodeModel::JITDefault; + JIT = true; + LLVM_FALLTHROUGH; + case LLVMCodeModelDefault: + return None; case LLVMCodeModelSmall: return CodeModel::Small; case LLVMCodeModelKernel: @@ -37,15 +40,11 @@ inline CodeModel::Model unwrap(LLVMCodeModel Model) { case LLVMCodeModelLarge: return CodeModel::Large; } - return CodeModel::Default; + return CodeModel::Small; } inline LLVMCodeModel wrap(CodeModel::Model Model) { switch (Model) { - case CodeModel::Default: - return LLVMCodeModelDefault; - case CodeModel::JITDefault: - return LLVMCodeModelJITDefault; case CodeModel::Small: return LLVMCodeModelSmall; case CodeModel::Kernel: @@ -61,4 +60,3 @@ inline LLVMCodeModel wrap(CodeModel::Model Model) { } // end llvm namespace #endif - diff --git a/contrib/llvm/include/llvm/Support/CodeGenCoverage.h b/contrib/llvm/include/llvm/Support/CodeGenCoverage.h new file mode 100644 index 000000000000..d5bd837bff28 --- /dev/null +++ b/contrib/llvm/include/llvm/Support/CodeGenCoverage.h @@ -0,0 +1,37 @@ +//== llvm/Support/CodeGenCoverage.h ------------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file This file provides rule coverage tracking for tablegen-erated CodeGen. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CODEGENCOVERAGE_H +#define LLVM_SUPPORT_CODEGENCOVERAGE_H + +#include "llvm/ADT/BitVector.h" + +namespace llvm { +class LLVMContext; +class MemoryBuffer; + +class CodeGenCoverage { +protected: + BitVector RuleCoverage; + +public: + CodeGenCoverage(); + + void setCovered(uint64_t RuleID); + bool isCovered(uint64_t RuleID); + + bool parse(MemoryBuffer &Buffer, StringRef BackendName); + bool emit(StringRef FilePrefix, StringRef BackendName) const; + void reset(); +}; +} // end namespace llvm + +#endif // ifndef LLVM_SUPPORT_CODEGENCOVERAGE_H diff --git a/contrib/llvm/include/llvm/Support/CommandLine.h b/contrib/llvm/include/llvm/Support/CommandLine.h index 71d2f0293083..d1901db7c68e 100644 --- a/contrib/llvm/include/llvm/Support/CommandLine.h +++ b/contrib/llvm/include/llvm/Support/CommandLine.h @@ -66,12 +66,15 @@ bool ParseCommandLineOptions(int argc, const char *const *argv, void ParseEnvironmentOptions(const char *progName, const char *envvar, const char *Overview = ""); +// Function pointer type for printing version information. +using VersionPrinterTy = std::function<void(raw_ostream &)>; + ///===---------------------------------------------------------------------===// /// SetVersionPrinter - Override the default (LLVM specific) version printer /// used to print out the version when --version is given /// on the command line. This allows other systems using the /// CommandLine utilities to print their own version string. -void SetVersionPrinter(void (*func)()); +void SetVersionPrinter(VersionPrinterTy func); ///===---------------------------------------------------------------------===// /// AddExtraVersionPrinter - Add an extra printer to use in addition to the @@ -80,7 +83,7 @@ void SetVersionPrinter(void (*func)()); /// which will be called after the basic LLVM version /// printing is complete. Each can then add additional /// information specific to the tool. -void AddExtraVersionPrinter(void (*func)()); +void AddExtraVersionPrinter(VersionPrinterTy func); // PrintOptionValues - Print option values. // With -print-options print the difference between option values and defaults. @@ -263,7 +266,7 @@ public: StringRef ValueStr; // String describing what the value of this option is OptionCategory *Category; // The Category this option belongs to SmallPtrSet<SubCommand *, 4> Subs; // The subcommands this option belongs to. - bool FullyInitialized = false; // Has addArguemnt been called? + bool FullyInitialized = false; // Has addArgument been called? inline enum NumOccurrencesFlag getNumOccurrencesFlag() const { return (enum NumOccurrencesFlag)Occurrences; @@ -346,6 +349,8 @@ public: virtual void printOptionValue(size_t GlobalWidth, bool Force) const = 0; + virtual void setDefault() = 0; + static void printHelpStr(StringRef HelpStr, size_t Indent, size_t FirstLineIndentedBy); @@ -1315,6 +1320,20 @@ class opt : public Option, } } + template <class T, class = typename std::enable_if< + std::is_assignable<T&, T>::value>::type> + void setDefaultImpl() { + const OptionValue<DataType> &V = this->getDefault(); + if (V.hasValue()) + this->setValue(V.getValue()); + } + + template <class T, class = typename std::enable_if< + !std::is_assignable<T&, T>::value>::type> + void setDefaultImpl(...) {} + + void setDefault() override { setDefaultImpl<DataType>(); } + void done() { addArgument(); Parser.initialize(); @@ -1490,6 +1509,8 @@ class list : public Option, public list_storage<DataType, StorageClass> { void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { } + void setDefault() override {} + void done() { addArgument(); Parser.initialize(); @@ -1631,6 +1652,8 @@ class bits : public Option, public bits_storage<DataType, Storage> { void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { } + void setDefault() override {} + void done() { addArgument(); Parser.initialize(); @@ -1681,6 +1704,8 @@ class alias : public Option { void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { } + void setDefault() override { AliasFor->setDefault(); } + ValueExpected getValueExpectedFlagDefault() const override { return AliasFor->getValueExpectedFlag(); } @@ -1737,8 +1762,6 @@ void PrintVersionMessage(); /// This function just prints the help message, exactly the same way as if the /// -help or -help-hidden option had been given on the command line. /// -/// NOTE: THIS FUNCTION TERMINATES THE PROGRAM! -/// /// \param Hidden if true will print hidden options /// \param Categorized if true print options in categories void PrintHelpMessage(bool Hidden = false, bool Categorized = false); diff --git a/contrib/llvm/include/llvm/Support/ConvertUTF.h b/contrib/llvm/include/llvm/Support/ConvertUTF.h index bd439f360216..99ae171aeabb 100644 --- a/contrib/llvm/include/llvm/Support/ConvertUTF.h +++ b/contrib/llvm/include/llvm/Support/ConvertUTF.h @@ -242,10 +242,10 @@ bool ConvertCodePointToUTF8(unsigned Source, char *&ResultPtr); * * \sa ConvertUTF8toUTF32 */ -static inline ConversionResult convertUTF8Sequence(const UTF8 **source, - const UTF8 *sourceEnd, - UTF32 *target, - ConversionFlags flags) { +inline ConversionResult convertUTF8Sequence(const UTF8 **source, + const UTF8 *sourceEnd, + UTF32 *target, + ConversionFlags flags) { if (*source == sourceEnd) return sourceExhausted; unsigned size = getNumBytesForUTF8(**source); diff --git a/contrib/llvm/include/llvm/Support/DebugCounter.h b/contrib/llvm/include/llvm/Support/DebugCounter.h index a533feae7fa3..52e1bd71a2f2 100644 --- a/contrib/llvm/include/llvm/Support/DebugCounter.h +++ b/contrib/llvm/include/llvm/Support/DebugCounter.h @@ -159,7 +159,7 @@ private: #define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC) \ static const unsigned VARNAME = \ - DebugCounter::registerCounter(COUNTERNAME, DESC); + DebugCounter::registerCounter(COUNTERNAME, DESC) } // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/Support/Error.h b/contrib/llvm/include/llvm/Support/Error.h index 9a7fa0ae6356..8567af392fb0 100644 --- a/contrib/llvm/include/llvm/Support/Error.h +++ b/contrib/llvm/include/llvm/Support/Error.h @@ -246,18 +246,20 @@ public: } private: +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + // assertIsChecked() happens very frequently, but under normal circumstances + // is supposed to be a no-op. So we want it to be inlined, but having a bunch + // of debug prints can cause the function to be too large for inlining. So + // it's important that we define this function out of line so that it can't be + // inlined. + LLVM_ATTRIBUTE_NORETURN + void fatalUncheckedError() const; +#endif + void assertIsChecked() { #if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (!getChecked() || getPtr()) { - dbgs() << "Program aborted due to an unhandled Error:\n"; - if (getPtr()) - getPtr()->log(dbgs()); - else - dbgs() - << "Error value was Success. (Note: Success values must still be " - "checked prior to being destroyed).\n"; - abort(); - } + if (LLVM_UNLIKELY(!getChecked() || getPtr())) + fatalUncheckedError(); #endif } @@ -407,235 +409,6 @@ inline Error joinErrors(Error E1, Error E2) { return ErrorList::join(std::move(E1), std::move(E2)); } -/// Helper for testing applicability of, and applying, handlers for -/// ErrorInfo types. -template <typename HandlerT> -class ErrorHandlerTraits - : public ErrorHandlerTraits<decltype( - &std::remove_reference<HandlerT>::type::operator())> {}; - -// Specialization functions of the form 'Error (const ErrT&)'. -template <typename ErrT> class ErrorHandlerTraits<Error (&)(ErrT &)> { -public: - static bool appliesTo(const ErrorInfoBase &E) { - return E.template isA<ErrT>(); - } - - template <typename HandlerT> - static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { - assert(appliesTo(*E) && "Applying incorrect handler"); - return H(static_cast<ErrT &>(*E)); - } -}; - -// Specialization functions of the form 'void (const ErrT&)'. -template <typename ErrT> class ErrorHandlerTraits<void (&)(ErrT &)> { -public: - static bool appliesTo(const ErrorInfoBase &E) { - return E.template isA<ErrT>(); - } - - template <typename HandlerT> - static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { - assert(appliesTo(*E) && "Applying incorrect handler"); - H(static_cast<ErrT &>(*E)); - return Error::success(); - } -}; - -/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'. -template <typename ErrT> -class ErrorHandlerTraits<Error (&)(std::unique_ptr<ErrT>)> { -public: - static bool appliesTo(const ErrorInfoBase &E) { - return E.template isA<ErrT>(); - } - - template <typename HandlerT> - static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { - assert(appliesTo(*E) && "Applying incorrect handler"); - std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release())); - return H(std::move(SubE)); - } -}; - -/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'. -template <typename ErrT> -class ErrorHandlerTraits<void (&)(std::unique_ptr<ErrT>)> { -public: - static bool appliesTo(const ErrorInfoBase &E) { - return E.template isA<ErrT>(); - } - - template <typename HandlerT> - static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { - assert(appliesTo(*E) && "Applying incorrect handler"); - std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release())); - H(std::move(SubE)); - return Error::success(); - } -}; - -// Specialization for member functions of the form 'RetT (const ErrT&)'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(ErrT &)> - : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; - -// Specialization for member functions of the form 'RetT (const ErrT&) const'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(ErrT &) const> - : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; - -// Specialization for member functions of the form 'RetT (const ErrT&)'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(const ErrT &)> - : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; - -// Specialization for member functions of the form 'RetT (const ErrT&) const'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(const ErrT &) const> - : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; - -/// Specialization for member functions of the form -/// 'RetT (std::unique_ptr<ErrT>) const'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>)> - : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {}; - -/// Specialization for member functions of the form -/// 'RetT (std::unique_ptr<ErrT>) const'. -template <typename C, typename RetT, typename ErrT> -class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>) const> - : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {}; - -inline Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload) { - return Error(std::move(Payload)); -} - -template <typename HandlerT, typename... HandlerTs> -Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload, - HandlerT &&Handler, HandlerTs &&... Handlers) { - if (ErrorHandlerTraits<HandlerT>::appliesTo(*Payload)) - return ErrorHandlerTraits<HandlerT>::apply(std::forward<HandlerT>(Handler), - std::move(Payload)); - return handleErrorImpl(std::move(Payload), - std::forward<HandlerTs>(Handlers)...); -} - -/// Pass the ErrorInfo(s) contained in E to their respective handlers. Any -/// unhandled errors (or Errors returned by handlers) are re-concatenated and -/// returned. -/// Because this function returns an error, its result must also be checked -/// or returned. If you intend to handle all errors use handleAllErrors -/// (which returns void, and will abort() on unhandled errors) instead. -template <typename... HandlerTs> -Error handleErrors(Error E, HandlerTs &&... Hs) { - if (!E) - return Error::success(); - - std::unique_ptr<ErrorInfoBase> Payload = E.takePayload(); - - if (Payload->isA<ErrorList>()) { - ErrorList &List = static_cast<ErrorList &>(*Payload); - Error R; - for (auto &P : List.Payloads) - R = ErrorList::join( - std::move(R), - handleErrorImpl(std::move(P), std::forward<HandlerTs>(Hs)...)); - return R; - } - - return handleErrorImpl(std::move(Payload), std::forward<HandlerTs>(Hs)...); -} - -/// Behaves the same as handleErrors, except that it requires that all -/// errors be handled by the given handlers. If any unhandled error remains -/// after the handlers have run, abort() will be called. -template <typename... HandlerTs> -void handleAllErrors(Error E, HandlerTs &&... Handlers) { - auto F = handleErrors(std::move(E), std::forward<HandlerTs>(Handlers)...); - // Cast 'F' to bool to set the 'Checked' flag if it's a success value: - (void)!F; -} - -/// Check that E is a non-error, then drop it. -inline void handleAllErrors(Error E) { - // Cast 'E' to a bool to set the 'Checked' flag if it's a success value: - (void)!E; -} - -/// Log all errors (if any) in E to OS. If there are any errors, ErrorBanner -/// will be printed before the first one is logged. A newline will be printed -/// after each error. -/// -/// This is useful in the base level of your program to allow clean termination -/// (allowing clean deallocation of resources, etc.), while reporting error -/// information to the user. -void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner); - -/// Write all error messages (if any) in E to a string. The newline character -/// is used to separate error messages. -inline std::string toString(Error E) { - SmallVector<std::string, 2> Errors; - handleAllErrors(std::move(E), [&Errors](const ErrorInfoBase &EI) { - Errors.push_back(EI.message()); - }); - return join(Errors.begin(), Errors.end(), "\n"); -} - -/// Consume a Error without doing anything. This method should be used -/// only where an error can be considered a reasonable and expected return -/// value. -/// -/// Uses of this method are potentially indicative of design problems: If it's -/// legitimate to do nothing while processing an "error", the error-producer -/// might be more clearly refactored to return an Optional<T>. -inline void consumeError(Error Err) { - handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {}); -} - -/// Helper for Errors used as out-parameters. -/// -/// This helper is for use with the Error-as-out-parameter idiom, where an error -/// is passed to a function or method by reference, rather than being returned. -/// In such cases it is helpful to set the checked bit on entry to the function -/// so that the error can be written to (unchecked Errors abort on assignment) -/// and clear the checked bit on exit so that clients cannot accidentally forget -/// to check the result. This helper performs these actions automatically using -/// RAII: -/// -/// @code{.cpp} -/// Result foo(Error &Err) { -/// ErrorAsOutParameter ErrAsOutParam(&Err); // 'Checked' flag set -/// // <body of foo> -/// // <- 'Checked' flag auto-cleared when ErrAsOutParam is destructed. -/// } -/// @endcode -/// -/// ErrorAsOutParameter takes an Error* rather than Error& so that it can be -/// used with optional Errors (Error pointers that are allowed to be null). If -/// ErrorAsOutParameter took an Error reference, an instance would have to be -/// created inside every condition that verified that Error was non-null. By -/// taking an Error pointer we can just create one instance at the top of the -/// function. -class ErrorAsOutParameter { -public: - ErrorAsOutParameter(Error *Err) : Err(Err) { - // Raise the checked bit if Err is success. - if (Err) - (void)!!*Err; - } - - ~ErrorAsOutParameter() { - // Clear the checked bit. - if (Err && !*Err) - *Err = Error::success(); - } - -private: - Error *Err; -}; - /// Tagged union holding either a T or a Error. /// /// This class parallels ErrorOr, but replaces error_code with Error. Since @@ -861,19 +634,26 @@ private: #endif } +#if LLVM_ENABLE_ABI_BREAKING_CHECKS + LLVM_ATTRIBUTE_NORETURN + LLVM_ATTRIBUTE_NOINLINE + void fatalUncheckedExpected() const { + dbgs() << "Expected<T> must be checked before access or destruction.\n"; + if (HasError) { + dbgs() << "Unchecked Expected<T> contained error:\n"; + (*getErrorStorage())->log(dbgs()); + } else + dbgs() << "Expected<T> value was in success state. (Note: Expected<T> " + "values in success mode must still be checked prior to being " + "destroyed).\n"; + abort(); + } +#endif + void assertIsChecked() { #if LLVM_ENABLE_ABI_BREAKING_CHECKS - if (Unchecked) { - dbgs() << "Expected<T> must be checked before access or destruction.\n"; - if (HasError) { - dbgs() << "Unchecked Expected<T> contained error:\n"; - (*getErrorStorage())->log(dbgs()); - } else - dbgs() << "Expected<T> value was in success state. (Note: Expected<T> " - "values in success mode must still be checked prior to being " - "destroyed).\n"; - abort(); - } + if (LLVM_UNLIKELY(Unchecked)) + fatalUncheckedExpected(); #endif } @@ -887,6 +667,344 @@ private: #endif }; +/// Report a serious error, calling any installed error handler. See +/// ErrorHandling.h. +LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, + bool gen_crash_diag = true); + +/// Report a fatal error if Err is a failure value. +/// +/// This function can be used to wrap calls to fallible functions ONLY when it +/// is known that the Error will always be a success value. E.g. +/// +/// @code{.cpp} +/// // foo only attempts the fallible operation if DoFallibleOperation is +/// // true. If DoFallibleOperation is false then foo always returns +/// // Error::success(). +/// Error foo(bool DoFallibleOperation); +/// +/// cantFail(foo(false)); +/// @endcode +inline void cantFail(Error Err, const char *Msg = nullptr) { + if (Err) { + if (!Msg) + Msg = "Failure value returned from cantFail wrapped call"; + llvm_unreachable(Msg); + } +} + +/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and +/// returns the contained value. +/// +/// This function can be used to wrap calls to fallible functions ONLY when it +/// is known that the Error will always be a success value. E.g. +/// +/// @code{.cpp} +/// // foo only attempts the fallible operation if DoFallibleOperation is +/// // true. If DoFallibleOperation is false then foo always returns an int. +/// Expected<int> foo(bool DoFallibleOperation); +/// +/// int X = cantFail(foo(false)); +/// @endcode +template <typename T> +T cantFail(Expected<T> ValOrErr, const char *Msg = nullptr) { + if (ValOrErr) + return std::move(*ValOrErr); + else { + if (!Msg) + Msg = "Failure value returned from cantFail wrapped call"; + llvm_unreachable(Msg); + } +} + +/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and +/// returns the contained reference. +/// +/// This function can be used to wrap calls to fallible functions ONLY when it +/// is known that the Error will always be a success value. E.g. +/// +/// @code{.cpp} +/// // foo only attempts the fallible operation if DoFallibleOperation is +/// // true. If DoFallibleOperation is false then foo always returns a Bar&. +/// Expected<Bar&> foo(bool DoFallibleOperation); +/// +/// Bar &X = cantFail(foo(false)); +/// @endcode +template <typename T> +T& cantFail(Expected<T&> ValOrErr, const char *Msg = nullptr) { + if (ValOrErr) + return *ValOrErr; + else { + if (!Msg) + Msg = "Failure value returned from cantFail wrapped call"; + llvm_unreachable(Msg); + } +} + +/// Helper for testing applicability of, and applying, handlers for +/// ErrorInfo types. +template <typename HandlerT> +class ErrorHandlerTraits + : public ErrorHandlerTraits<decltype( + &std::remove_reference<HandlerT>::type::operator())> {}; + +// Specialization functions of the form 'Error (const ErrT&)'. +template <typename ErrT> class ErrorHandlerTraits<Error (&)(ErrT &)> { +public: + static bool appliesTo(const ErrorInfoBase &E) { + return E.template isA<ErrT>(); + } + + template <typename HandlerT> + static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { + assert(appliesTo(*E) && "Applying incorrect handler"); + return H(static_cast<ErrT &>(*E)); + } +}; + +// Specialization functions of the form 'void (const ErrT&)'. +template <typename ErrT> class ErrorHandlerTraits<void (&)(ErrT &)> { +public: + static bool appliesTo(const ErrorInfoBase &E) { + return E.template isA<ErrT>(); + } + + template <typename HandlerT> + static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { + assert(appliesTo(*E) && "Applying incorrect handler"); + H(static_cast<ErrT &>(*E)); + return Error::success(); + } +}; + +/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'. +template <typename ErrT> +class ErrorHandlerTraits<Error (&)(std::unique_ptr<ErrT>)> { +public: + static bool appliesTo(const ErrorInfoBase &E) { + return E.template isA<ErrT>(); + } + + template <typename HandlerT> + static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { + assert(appliesTo(*E) && "Applying incorrect handler"); + std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release())); + return H(std::move(SubE)); + } +}; + +/// Specialization for functions of the form 'void (std::unique_ptr<ErrT>)'. +template <typename ErrT> +class ErrorHandlerTraits<void (&)(std::unique_ptr<ErrT>)> { +public: + static bool appliesTo(const ErrorInfoBase &E) { + return E.template isA<ErrT>(); + } + + template <typename HandlerT> + static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) { + assert(appliesTo(*E) && "Applying incorrect handler"); + std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release())); + H(std::move(SubE)); + return Error::success(); + } +}; + +// Specialization for member functions of the form 'RetT (const ErrT&)'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(ErrT &)> + : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; + +// Specialization for member functions of the form 'RetT (const ErrT&) const'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(ErrT &) const> + : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; + +// Specialization for member functions of the form 'RetT (const ErrT&)'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(const ErrT &)> + : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; + +// Specialization for member functions of the form 'RetT (const ErrT&) const'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(const ErrT &) const> + : public ErrorHandlerTraits<RetT (&)(ErrT &)> {}; + +/// Specialization for member functions of the form +/// 'RetT (std::unique_ptr<ErrT>)'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>)> + : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {}; + +/// Specialization for member functions of the form +/// 'RetT (std::unique_ptr<ErrT>) const'. +template <typename C, typename RetT, typename ErrT> +class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>) const> + : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {}; + +inline Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload) { + return Error(std::move(Payload)); +} + +template <typename HandlerT, typename... HandlerTs> +Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload, + HandlerT &&Handler, HandlerTs &&... Handlers) { + if (ErrorHandlerTraits<HandlerT>::appliesTo(*Payload)) + return ErrorHandlerTraits<HandlerT>::apply(std::forward<HandlerT>(Handler), + std::move(Payload)); + return handleErrorImpl(std::move(Payload), + std::forward<HandlerTs>(Handlers)...); +} + +/// Pass the ErrorInfo(s) contained in E to their respective handlers. Any +/// unhandled errors (or Errors returned by handlers) are re-concatenated and +/// returned. +/// Because this function returns an error, its result must also be checked +/// or returned. If you intend to handle all errors use handleAllErrors +/// (which returns void, and will abort() on unhandled errors) instead. +template <typename... HandlerTs> +Error handleErrors(Error E, HandlerTs &&... Hs) { + if (!E) + return Error::success(); + + std::unique_ptr<ErrorInfoBase> Payload = E.takePayload(); + + if (Payload->isA<ErrorList>()) { + ErrorList &List = static_cast<ErrorList &>(*Payload); + Error R; + for (auto &P : List.Payloads) + R = ErrorList::join( + std::move(R), + handleErrorImpl(std::move(P), std::forward<HandlerTs>(Hs)...)); + return R; + } + + return handleErrorImpl(std::move(Payload), std::forward<HandlerTs>(Hs)...); +} + +/// Behaves the same as handleErrors, except that it requires that all +/// errors be handled by the given handlers. If any unhandled error remains +/// after the handlers have run, report_fatal_error() will be called. +template <typename... HandlerTs> +void handleAllErrors(Error E, HandlerTs &&... Handlers) { + cantFail(handleErrors(std::move(E), std::forward<HandlerTs>(Handlers)...)); +} + +/// Check that E is a non-error, then drop it. +/// If E is an error report_fatal_error will be called. +inline void handleAllErrors(Error E) { + cantFail(std::move(E)); +} + +/// Handle any errors (if present) in an Expected<T>, then try a recovery path. +/// +/// If the incoming value is a success value it is returned unmodified. If it +/// is a failure value then it the contained error is passed to handleErrors. +/// If handleErrors is able to handle the error then the RecoveryPath functor +/// is called to supply the final result. If handleErrors is not able to +/// handle all errors then the unhandled errors are returned. +/// +/// This utility enables the follow pattern: +/// +/// @code{.cpp} +/// enum FooStrategy { Aggressive, Conservative }; +/// Expected<Foo> foo(FooStrategy S); +/// +/// auto ResultOrErr = +/// handleExpected( +/// foo(Aggressive), +/// []() { return foo(Conservative); }, +/// [](AggressiveStrategyError&) { +/// // Implicitly conusme this - we'll recover by using a conservative +/// // strategy. +/// }); +/// +/// @endcode +template <typename T, typename RecoveryFtor, typename... HandlerTs> +Expected<T> handleExpected(Expected<T> ValOrErr, RecoveryFtor &&RecoveryPath, + HandlerTs &&... Handlers) { + if (ValOrErr) + return ValOrErr; + + if (auto Err = handleErrors(ValOrErr.takeError(), + std::forward<HandlerTs>(Handlers)...)) + return std::move(Err); + + return RecoveryPath(); +} + +/// Log all errors (if any) in E to OS. If there are any errors, ErrorBanner +/// will be printed before the first one is logged. A newline will be printed +/// after each error. +/// +/// This is useful in the base level of your program to allow clean termination +/// (allowing clean deallocation of resources, etc.), while reporting error +/// information to the user. +void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner); + +/// Write all error messages (if any) in E to a string. The newline character +/// is used to separate error messages. +inline std::string toString(Error E) { + SmallVector<std::string, 2> Errors; + handleAllErrors(std::move(E), [&Errors](const ErrorInfoBase &EI) { + Errors.push_back(EI.message()); + }); + return join(Errors.begin(), Errors.end(), "\n"); +} + +/// Consume a Error without doing anything. This method should be used +/// only where an error can be considered a reasonable and expected return +/// value. +/// +/// Uses of this method are potentially indicative of design problems: If it's +/// legitimate to do nothing while processing an "error", the error-producer +/// might be more clearly refactored to return an Optional<T>. +inline void consumeError(Error Err) { + handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {}); +} + +/// Helper for Errors used as out-parameters. +/// +/// This helper is for use with the Error-as-out-parameter idiom, where an error +/// is passed to a function or method by reference, rather than being returned. +/// In such cases it is helpful to set the checked bit on entry to the function +/// so that the error can be written to (unchecked Errors abort on assignment) +/// and clear the checked bit on exit so that clients cannot accidentally forget +/// to check the result. This helper performs these actions automatically using +/// RAII: +/// +/// @code{.cpp} +/// Result foo(Error &Err) { +/// ErrorAsOutParameter ErrAsOutParam(&Err); // 'Checked' flag set +/// // <body of foo> +/// // <- 'Checked' flag auto-cleared when ErrAsOutParam is destructed. +/// } +/// @endcode +/// +/// ErrorAsOutParameter takes an Error* rather than Error& so that it can be +/// used with optional Errors (Error pointers that are allowed to be null). If +/// ErrorAsOutParameter took an Error reference, an instance would have to be +/// created inside every condition that verified that Error was non-null. By +/// taking an Error pointer we can just create one instance at the top of the +/// function. +class ErrorAsOutParameter { +public: + ErrorAsOutParameter(Error *Err) : Err(Err) { + // Raise the checked bit if Err is success. + if (Err) + (void)!!*Err; + } + + ~ErrorAsOutParameter() { + // Clear the checked bit. + if (Err && !*Err) + *Err = Error::success(); + } + +private: + Error *Err; +}; + /// Helper for Expected<T>s used as out-parameters. /// /// See ErrorAsOutParameter. @@ -1032,71 +1150,6 @@ private: std::function<int(const Error &)> GetExitCode; }; -/// Report a serious error, calling any installed error handler. See -/// ErrorHandling.h. -LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, - bool gen_crash_diag = true); - -/// Report a fatal error if Err is a failure value. -/// -/// This function can be used to wrap calls to fallible functions ONLY when it -/// is known that the Error will always be a success value. E.g. -/// -/// @code{.cpp} -/// // foo only attempts the fallible operation if DoFallibleOperation is -/// // true. If DoFallibleOperation is false then foo always returns -/// // Error::success(). -/// Error foo(bool DoFallibleOperation); -/// -/// cantFail(foo(false)); -/// @endcode -inline void cantFail(Error Err) { - if (Err) - llvm_unreachable("Failure value returned from cantFail wrapped call"); -} - -/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and -/// returns the contained value. -/// -/// This function can be used to wrap calls to fallible functions ONLY when it -/// is known that the Error will always be a success value. E.g. -/// -/// @code{.cpp} -/// // foo only attempts the fallible operation if DoFallibleOperation is -/// // true. If DoFallibleOperation is false then foo always returns an int. -/// Expected<int> foo(bool DoFallibleOperation); -/// -/// int X = cantFail(foo(false)); -/// @endcode -template <typename T> -T cantFail(Expected<T> ValOrErr) { - if (ValOrErr) - return std::move(*ValOrErr); - else - llvm_unreachable("Failure value returned from cantFail wrapped call"); -} - -/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and -/// returns the contained reference. -/// -/// This function can be used to wrap calls to fallible functions ONLY when it -/// is known that the Error will always be a success value. E.g. -/// -/// @code{.cpp} -/// // foo only attempts the fallible operation if DoFallibleOperation is -/// // true. If DoFallibleOperation is false then foo always returns a Bar&. -/// Expected<Bar&> foo(bool DoFallibleOperation); -/// -/// Bar &X = cantFail(foo(false)); -/// @endcode -template <typename T> -T& cantFail(Expected<T&> ValOrErr) { - if (ValOrErr) - return *ValOrErr; - else - llvm_unreachable("Failure value returned from cantFail wrapped call"); -} - } // end namespace llvm #endif // LLVM_SUPPORT_ERROR_H diff --git a/contrib/llvm/include/llvm/Support/FileOutputBuffer.h b/contrib/llvm/include/llvm/Support/FileOutputBuffer.h index 2c054c75374b..6aed423a01e3 100644 --- a/contrib/llvm/include/llvm/Support/FileOutputBuffer.h +++ b/contrib/llvm/include/llvm/Support/FileOutputBuffer.h @@ -17,7 +17,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" -#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" namespace llvm { @@ -30,7 +30,6 @@ namespace llvm { /// not committed, the file will be deleted in the FileOutputBuffer destructor. class FileOutputBuffer { public: - enum { F_executable = 1 /// set the 'x' bit on the resulting file }; @@ -38,52 +37,37 @@ public: /// Factory method to create an OutputBuffer object which manages a read/write /// buffer of the specified size. When committed, the buffer will be written /// to the file at the specified path. - static ErrorOr<std::unique_ptr<FileOutputBuffer>> + static Expected<std::unique_ptr<FileOutputBuffer>> create(StringRef FilePath, size_t Size, unsigned Flags = 0); /// Returns a pointer to the start of the buffer. - uint8_t *getBufferStart() { - return (uint8_t*)Region->data(); - } + virtual uint8_t *getBufferStart() const = 0; /// Returns a pointer to the end of the buffer. - uint8_t *getBufferEnd() { - return (uint8_t*)Region->data() + Region->size(); - } + virtual uint8_t *getBufferEnd() const = 0; /// Returns size of the buffer. - size_t getBufferSize() const { - return Region->size(); - } + virtual size_t getBufferSize() const = 0; /// Returns path where file will show up if buffer is committed. - StringRef getPath() const { - return FinalPath; - } + StringRef getPath() const { return FinalPath; } /// Flushes the content of the buffer to its file and deallocates the /// buffer. If commit() is not called before this object's destructor /// is called, the file is deleted in the destructor. The optional parameter /// is used if it turns out you want the file size to be smaller than /// initially requested. - std::error_code commit(); + virtual Error commit() = 0; /// If this object was previously committed, the destructor just deletes /// this object. If this object was not committed, the destructor /// deallocates the buffer and the target file is never written. - ~FileOutputBuffer(); - -private: - FileOutputBuffer(const FileOutputBuffer &) = delete; - FileOutputBuffer &operator=(const FileOutputBuffer &) = delete; + virtual ~FileOutputBuffer() {} - FileOutputBuffer(std::unique_ptr<llvm::sys::fs::mapped_file_region> R, - StringRef Path, StringRef TempPath, bool IsRegular); +protected: + FileOutputBuffer(StringRef Path) : FinalPath(Path) {} - std::unique_ptr<llvm::sys::fs::mapped_file_region> Region; - SmallString<128> FinalPath; - SmallString<128> TempPath; - bool IsRegular; + std::string FinalPath; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Support/FileSystem.h b/contrib/llvm/include/llvm/Support/FileSystem.h index 21c5fcdb7145..b1683ba5ddb3 100644 --- a/contrib/llvm/include/llvm/Support/FileSystem.h +++ b/contrib/llvm/include/llvm/Support/FileSystem.h @@ -31,6 +31,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/Chrono.h" +#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MD5.h" @@ -141,65 +142,48 @@ public: uint64_t getFile() const { return File; } }; -/// file_status - Represents the result of a call to stat and friends. It has -/// a platform-specific member to store the result. -class file_status -{ - friend bool equivalent(file_status A, file_status B); - +/// Represents the result of a call to directory_iterator::status(). This is a +/// subset of the information returned by a regular sys::fs::status() call, and +/// represents the information provided by Windows FileFirstFile/FindNextFile. +class basic_file_status { +protected: #if defined(LLVM_ON_UNIX) - dev_t fs_st_dev = 0; - nlink_t fs_st_nlinks = 0; - ino_t fs_st_ino = 0; time_t fs_st_atime = 0; time_t fs_st_mtime = 0; uid_t fs_st_uid = 0; gid_t fs_st_gid = 0; off_t fs_st_size = 0; #elif defined (LLVM_ON_WIN32) - uint32_t NumLinks = 0; uint32_t LastAccessedTimeHigh = 0; uint32_t LastAccessedTimeLow = 0; uint32_t LastWriteTimeHigh = 0; uint32_t LastWriteTimeLow = 0; - uint32_t VolumeSerialNumber = 0; uint32_t FileSizeHigh = 0; uint32_t FileSizeLow = 0; - uint32_t FileIndexHigh = 0; - uint32_t FileIndexLow = 0; #endif file_type Type = file_type::status_error; perms Perms = perms_not_known; public: - #if defined(LLVM_ON_UNIX) - file_status() = default; + basic_file_status() = default; - file_status(file_type Type) : Type(Type) {} + explicit basic_file_status(file_type Type) : Type(Type) {} - file_status(file_type Type, perms Perms, dev_t Dev, nlink_t Links, ino_t Ino, - time_t ATime, time_t MTime, uid_t UID, gid_t GID, off_t Size) - : fs_st_dev(Dev), fs_st_nlinks(Links), fs_st_ino(Ino), fs_st_atime(ATime), - fs_st_mtime(MTime), fs_st_uid(UID), fs_st_gid(GID), fs_st_size(Size), - Type(Type), Perms(Perms) {} - #elif defined(LLVM_ON_WIN32) - file_status() = default; - - file_status(file_type Type) : Type(Type) {} - - file_status(file_type Type, perms Perms, uint32_t LinkCount, - uint32_t LastAccessTimeHigh, uint32_t LastAccessTimeLow, - uint32_t LastWriteTimeHigh, uint32_t LastWriteTimeLow, - uint32_t VolumeSerialNumber, uint32_t FileSizeHigh, - uint32_t FileSizeLow, uint32_t FileIndexHigh, - uint32_t FileIndexLow) - : NumLinks(LinkCount), LastAccessedTimeHigh(LastAccessTimeHigh), + #if defined(LLVM_ON_UNIX) + basic_file_status(file_type Type, perms Perms, time_t ATime, time_t MTime, + uid_t UID, gid_t GID, off_t Size) + : fs_st_atime(ATime), fs_st_mtime(MTime), fs_st_uid(UID), fs_st_gid(GID), + fs_st_size(Size), Type(Type), Perms(Perms) {} +#elif defined(LLVM_ON_WIN32) + basic_file_status(file_type Type, perms Perms, uint32_t LastAccessTimeHigh, + uint32_t LastAccessTimeLow, uint32_t LastWriteTimeHigh, + uint32_t LastWriteTimeLow, uint32_t FileSizeHigh, + uint32_t FileSizeLow) + : LastAccessedTimeHigh(LastAccessTimeHigh), LastAccessedTimeLow(LastAccessTimeLow), LastWriteTimeHigh(LastWriteTimeHigh), - LastWriteTimeLow(LastWriteTimeLow), - VolumeSerialNumber(VolumeSerialNumber), FileSizeHigh(FileSizeHigh), - FileSizeLow(FileSizeLow), FileIndexHigh(FileIndexHigh), - FileIndexLow(FileIndexLow), Type(Type), Perms(Perms) {} + LastWriteTimeLow(LastWriteTimeLow), FileSizeHigh(FileSizeHigh), + FileSizeLow(FileSizeLow), Type(Type), Perms(Perms) {} #endif // getters @@ -207,8 +191,6 @@ public: perms permissions() const { return Perms; } TimePoint<> getLastAccessedTime() const; TimePoint<> getLastModificationTime() const; - UniqueID getUniqueID() const; - uint32_t getLinkCount() const; #if defined(LLVM_ON_UNIX) uint32_t getUser() const { return fs_st_uid; } @@ -233,6 +215,49 @@ public: void permissions(perms p) { Perms = p; } }; +/// Represents the result of a call to sys::fs::status(). +class file_status : public basic_file_status { + friend bool equivalent(file_status A, file_status B); + + #if defined(LLVM_ON_UNIX) + dev_t fs_st_dev = 0; + nlink_t fs_st_nlinks = 0; + ino_t fs_st_ino = 0; + #elif defined (LLVM_ON_WIN32) + uint32_t NumLinks = 0; + uint32_t VolumeSerialNumber = 0; + uint32_t FileIndexHigh = 0; + uint32_t FileIndexLow = 0; + #endif + +public: + file_status() = default; + + explicit file_status(file_type Type) : basic_file_status(Type) {} + + #if defined(LLVM_ON_UNIX) + file_status(file_type Type, perms Perms, dev_t Dev, nlink_t Links, ino_t Ino, + time_t ATime, time_t MTime, uid_t UID, gid_t GID, off_t Size) + : basic_file_status(Type, Perms, ATime, MTime, UID, GID, Size), + fs_st_dev(Dev), fs_st_nlinks(Links), fs_st_ino(Ino) {} + #elif defined(LLVM_ON_WIN32) + file_status(file_type Type, perms Perms, uint32_t LinkCount, + uint32_t LastAccessTimeHigh, uint32_t LastAccessTimeLow, + uint32_t LastWriteTimeHigh, uint32_t LastWriteTimeLow, + uint32_t VolumeSerialNumber, uint32_t FileSizeHigh, + uint32_t FileSizeLow, uint32_t FileIndexHigh, + uint32_t FileIndexLow) + : basic_file_status(Type, Perms, LastAccessTimeHigh, LastAccessTimeLow, + LastWriteTimeHigh, LastWriteTimeLow, FileSizeHigh, + FileSizeLow), + NumLinks(LinkCount), VolumeSerialNumber(VolumeSerialNumber), + FileIndexHigh(FileIndexHigh), FileIndexLow(FileIndexLow) {} + #endif + + UniqueID getUniqueID() const; + uint32_t getLinkCount() const; +}; + /// @} /// @name Physical Operators /// @{ @@ -343,7 +368,11 @@ std::error_code remove(const Twine &path, bool IgnoreNonExisting = true); /// platform-specific error code. std::error_code remove_directories(const Twine &path, bool IgnoreErrors = true); -/// @brief Rename \a from to \a to. Files are renamed as if by POSIX rename(). +/// @brief Rename \a from to \a to. +/// +/// Files are renamed as if by POSIX rename(), except that on Windows there may +/// be a short interval of time during which the destination file does not +/// exist. /// /// @param from The path to rename from. /// @param to The path to rename to. This is created. @@ -379,10 +408,10 @@ ErrorOr<MD5::MD5Result> md5_contents(const Twine &Path); /// @brief Does file exist? /// -/// @param status A file_status previously returned from stat. +/// @param status A basic_file_status previously returned from stat. /// @returns True if the file represented by status exists, false if it does /// not. -bool exists(file_status status); +bool exists(const basic_file_status &status); enum class AccessMode { Exist, Write, Execute }; @@ -481,9 +510,9 @@ file_type get_file_type(const Twine &Path, bool Follow = true); /// @brief Does status represent a directory? /// -/// @param status A file_status previously returned from status. +/// @param status A basic_file_status previously returned from status. /// @returns status.type() == file_type::directory_file. -bool is_directory(file_status status); +bool is_directory(const basic_file_status &status); /// @brief Is path a directory? /// @@ -503,9 +532,9 @@ inline bool is_directory(const Twine &Path) { /// @brief Does status represent a regular file? /// -/// @param status A file_status previously returned from status. +/// @param status A basic_file_status previously returned from status. /// @returns status_known(status) && status.type() == file_type::regular_file. -bool is_regular_file(file_status status); +bool is_regular_file(const basic_file_status &status); /// @brief Is path a regular file? /// @@ -527,9 +556,9 @@ inline bool is_regular_file(const Twine &Path) { /// @brief Does status represent a symlink file? /// -/// @param status A file_status previously returned from status. +/// @param status A basic_file_status previously returned from status. /// @returns status_known(status) && status.type() == file_type::symlink_file. -bool is_symlink_file(file_status status); +bool is_symlink_file(const basic_file_status &status); /// @brief Is path a symlink file? /// @@ -552,9 +581,9 @@ inline bool is_symlink_file(const Twine &Path) { /// @brief Does this status represent something that exists but is not a /// directory or regular file? /// -/// @param status A file_status previously returned from status. +/// @param status A basic_file_status previously returned from status. /// @returns exists(s) && !is_regular_file(s) && !is_directory(s) -bool is_other(file_status status); +bool is_other(const basic_file_status &status); /// @brief Is path something that exists but is not a directory, /// regular file, or symlink? @@ -627,7 +656,7 @@ std::error_code setLastModificationAndAccessTime(int FD, TimePoint<> Time); /// /// @param s Input file status. /// @returns True if status() != status_error. -bool status_known(file_status s); +bool status_known(const basic_file_status &s); /// @brief Is status available? /// @@ -637,6 +666,29 @@ bool status_known(file_status s); /// platform-specific error_code. std::error_code status_known(const Twine &path, bool &result); +enum OpenFlags : unsigned { + F_None = 0, + + /// F_Excl - When opening a file, this flag makes raw_fd_ostream + /// report an error if the file already exists. + F_Excl = 1, + + /// F_Append - When opening a file, if it already exists append to the + /// existing file instead of returning an error. This may not be specified + /// with F_Excl. + F_Append = 2, + + /// The file should be opened in text mode on platforms that make this + /// distinction. + F_Text = 4, + + /// Open the file for read and write. + F_RW = 8, + + /// Delete the file on close. Only makes a difference on windows. + F_Delete = 16 +}; + /// @brief Create a uniquely named file. /// /// Generates a unique path suitable for a temporary file and then opens it as a @@ -660,12 +712,51 @@ std::error_code status_known(const Twine &path, bool &result); /// otherwise a platform-specific error_code. std::error_code createUniqueFile(const Twine &Model, int &ResultFD, SmallVectorImpl<char> &ResultPath, - unsigned Mode = all_read | all_write); + unsigned Mode = all_read | all_write, + sys::fs::OpenFlags Flags = sys::fs::F_RW); /// @brief Simpler version for clients that don't want an open file. std::error_code createUniqueFile(const Twine &Model, SmallVectorImpl<char> &ResultPath); +/// Represents a temporary file. +/// +/// The temporary file must be eventually discarded or given a final name and +/// kept. +/// +/// The destructor doesn't implicitly discard because there is no way to +/// properly handle errors in a destructor. +class TempFile { + bool Done = false; + TempFile(StringRef Name, int FD); + +public: + /// This creates a temporary file with createUniqueFile and schedules it for + /// deletion with sys::RemoveFileOnSignal. + static Expected<TempFile> create(const Twine &Model, + unsigned Mode = all_read | all_write); + TempFile(TempFile &&Other); + TempFile &operator=(TempFile &&Other); + + // Name of the temporary file. + std::string TmpName; + + // The open file descriptor. + int FD = -1; + + // Keep this with the given name. + Error keep(const Twine &Name); + + // Keep this with the temporary name. + Error keep(); + + // Delete the file. + Error discard(); + + // This checks that keep or delete was called. + ~TempFile(); +}; + /// @brief Create a file in the system temporary directory. /// /// The filename is of the form prefix-random_chars.suffix. Since the directory @@ -676,7 +767,8 @@ std::error_code createUniqueFile(const Twine &Model, /// running the assembler. std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix, int &ResultFD, - SmallVectorImpl<char> &ResultPath); + SmallVectorImpl<char> &ResultPath, + sys::fs::OpenFlags Flags = sys::fs::F_RW); /// @brief Simpler version for clients that don't want an open file. std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix, @@ -685,32 +777,6 @@ std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix, std::error_code createUniqueDirectory(const Twine &Prefix, SmallVectorImpl<char> &ResultPath); -/// @brief Fetch a path to an open file, as specified by a file descriptor -/// -/// @param FD File descriptor to a currently open file -/// @param ResultPath The buffer into which to write the path -std::error_code getPathFromOpenFD(int FD, SmallVectorImpl<char> &ResultPath); - -enum OpenFlags : unsigned { - F_None = 0, - - /// F_Excl - When opening a file, this flag makes raw_fd_ostream - /// report an error if the file already exists. - F_Excl = 1, - - /// F_Append - When opening a file, if it already exists append to the - /// existing file instead of returning an error. This may not be specified - /// with F_Excl. - F_Append = 2, - - /// The file should be opened in text mode on platforms that make this - /// distinction. - F_Text = 4, - - /// Open the file for read and write. - F_RW = 8 -}; - inline OpenFlags operator|(OpenFlags A, OpenFlags B) { return OpenFlags(unsigned(A) | unsigned(B)); } @@ -751,7 +817,7 @@ public: private: /// Platform-specific mapping state. - uint64_t Size; + size_t Size; void *Mapping; std::error_code init(int FD, uint64_t Offset, mapmode Mode); @@ -764,12 +830,12 @@ public: /// \param fd An open file descriptor to map. mapped_file_region takes /// ownership if closefd is true. It must have been opended in the correct /// mode. - mapped_file_region(int fd, mapmode mode, uint64_t length, uint64_t offset, + mapped_file_region(int fd, mapmode mode, size_t length, uint64_t offset, std::error_code &ec); ~mapped_file_region(); - uint64_t size() const; + size_t size() const; char *data() const; /// Get a const view of the data. Modifying this memory has undefined @@ -795,24 +861,25 @@ std::string getMainExecutable(const char *argv0, void *MainExecAddr); class directory_entry { std::string Path; bool FollowSymlinks; - mutable file_status Status; + basic_file_status Status; public: explicit directory_entry(const Twine &path, bool follow_symlinks = true, - file_status st = file_status()) + basic_file_status st = basic_file_status()) : Path(path.str()), FollowSymlinks(follow_symlinks), Status(st) {} directory_entry() = default; - void assign(const Twine &path, file_status st = file_status()) { + void assign(const Twine &path, basic_file_status st = basic_file_status()) { Path = path.str(); Status = st; } - void replace_filename(const Twine &filename, file_status st = file_status()); + void replace_filename(const Twine &filename, + basic_file_status st = basic_file_status()); const std::string &path() const { return Path; } - std::error_code status(file_status &result) const; + ErrorOr<basic_file_status> status() const; bool operator==(const directory_entry& rhs) const { return Path == rhs.Path; } bool operator!=(const directory_entry& rhs) const { return !(*this == rhs); } @@ -931,9 +998,9 @@ public: if (State->HasNoPushRequest) State->HasNoPushRequest = false; else { - file_status st; - if ((ec = State->Stack.top()->status(st))) return *this; - if (is_directory(st)) { + ErrorOr<basic_file_status> st = State->Stack.top()->status(); + if (!st) return *this; + if (is_directory(*st)) { State->Stack.push(directory_iterator(*State->Stack.top(), ec, Follow)); if (ec) return *this; if (State->Stack.top() != end_itr) { diff --git a/contrib/llvm/include/llvm/Support/FormatVariadic.h b/contrib/llvm/include/llvm/Support/FormatVariadic.h index 408c6d8b2e0d..8c08a7d9488f 100644 --- a/contrib/llvm/include/llvm/Support/FormatVariadic.h +++ b/contrib/llvm/include/llvm/Support/FormatVariadic.h @@ -230,9 +230,8 @@ public: // For a given parameter of type T, the following steps are executed in order // until a match is found: // -// 1. If the parameter is of class type, and contains a method -// void format(raw_ostream &Stream, StringRef Options) -// Then this method is invoked to produce the formatted output. The +// 1. If the parameter is of class type, and inherits from format_adapter, +// Then format() is invoked on it to produce the formatted output. The // implementation should write the formatted text into `Stream`. // 2. If there is a suitable template specialization of format_provider<> // for type T containing a method whose signature is: @@ -259,6 +258,13 @@ inline auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object<decltype( std::make_tuple(detail::build_format_adapter(std::forward<Ts>(Vals))...)); } +// Allow a formatv_object to be formatted (no options supported). +template <typename T> struct format_provider<formatv_object<T>> { + static void format(const formatv_object<T> &V, raw_ostream &OS, StringRef) { + OS << V; + } +}; + } // end namespace llvm #endif // LLVM_SUPPORT_FORMATVARIADIC_H diff --git a/contrib/llvm/include/llvm/Support/FormatVariadicDetails.h b/contrib/llvm/include/llvm/Support/FormatVariadicDetails.h index b4a564ffc26c..9b60462209dc 100644 --- a/contrib/llvm/include/llvm/Support/FormatVariadicDetails.h +++ b/contrib/llvm/include/llvm/Support/FormatVariadicDetails.h @@ -31,7 +31,7 @@ template <typename T> class provider_format_adapter : public format_adapter { T Item; public: - explicit provider_format_adapter(T &&Item) : Item(Item) {} + explicit provider_format_adapter(T &&Item) : Item(std::forward<T>(Item)) {} void format(llvm::raw_ostream &S, StringRef Options) override { format_provider<typename std::decay<T>::type>::format(Item, S, Options); diff --git a/contrib/llvm/include/llvm/Support/GenericDomTree.h b/contrib/llvm/include/llvm/Support/GenericDomTree.h index 706320fed9a7..635c87a106f0 100644 --- a/contrib/llvm/include/llvm/Support/GenericDomTree.h +++ b/contrib/llvm/include/llvm/Support/GenericDomTree.h @@ -14,8 +14,8 @@ /// graph types. /// /// Unlike ADT/* graph algorithms, generic dominator tree has more requirements -/// on the graph's NodeRef. The NodeRef should be a pointer and, depending on -/// the implementation, e.g. NodeRef->getParent() return the parent node. +/// on the graph's NodeRef. The NodeRef should be a pointer and, +/// NodeRef->getParent() must return the parent node that is also a pointer. /// /// FIXME: Maybe GenericDomTree needs a TreeTraits, instead of GraphTraits. /// @@ -24,12 +24,6 @@ #ifndef LLVM_SUPPORT_GENERICDOMTREE_H #define LLVM_SUPPORT_GENERICDOMTREE_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/GraphTraits.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -38,6 +32,13 @@ #include <type_traits> #include <utility> #include <vector> +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/PointerIntPair.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/raw_ostream.h" namespace llvm { @@ -45,7 +46,7 @@ template <typename NodeT, bool IsPostDom> class DominatorTreeBase; namespace DomTreeBuilder { -template <class DomTreeT> +template <typename DomTreeT> struct SemiNCAInfo; } // namespace DomTreeBuilder @@ -187,17 +188,51 @@ void PrintDomTree(const DomTreeNodeBase<NodeT> *N, raw_ostream &O, namespace DomTreeBuilder { // The routines below are provided in a separate header but referenced here. -template <typename DomTreeT, typename FuncT> -void Calculate(DomTreeT &DT, FuncT &F); +template <typename DomTreeT> +void Calculate(DomTreeT &DT); -template <class DomTreeT> +template <typename DomTreeT> void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, typename DomTreeT::NodePtr To); -template <class DomTreeT> +template <typename DomTreeT> void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, typename DomTreeT::NodePtr To); +// UpdateKind and Update are used by the batch update API and it's easiest to +// define them here. +enum class UpdateKind : unsigned char { Insert, Delete }; + +template <typename NodePtr> +struct Update { + using NodeKindPair = PointerIntPair<NodePtr, 1, UpdateKind>; + + NodePtr From; + NodeKindPair ToAndKind; + + Update(UpdateKind Kind, NodePtr From, NodePtr To) + : From(From), ToAndKind(To, Kind) {} + + UpdateKind getKind() const { return ToAndKind.getInt(); } + NodePtr getFrom() const { return From; } + NodePtr getTo() const { return ToAndKind.getPointer(); } + bool operator==(const Update &RHS) const { + return From == RHS.From && ToAndKind == RHS.ToAndKind; + } + + friend raw_ostream &operator<<(raw_ostream &OS, const Update &U) { + OS << (U.getKind() == UpdateKind::Insert ? "Insert " : "Delete "); + U.getFrom()->printAsOperand(OS, false); + OS << " -> "; + U.getTo()->printAsOperand(OS, false); + return OS; + } +}; + +template <typename DomTreeT> +void ApplyUpdates(DomTreeT &DT, + ArrayRef<typename DomTreeT::UpdateType> Updates); + template <typename DomTreeT> bool Verify(const DomTreeT &DT); } // namespace DomTreeBuilder @@ -208,14 +243,30 @@ bool Verify(const DomTreeT &DT); /// various graphs in the LLVM IR or in the code generator. template <typename NodeT, bool IsPostDom> class DominatorTreeBase { + public: + static_assert(std::is_pointer<typename GraphTraits<NodeT *>::NodeRef>::value, + "Currently DominatorTreeBase supports only pointer nodes"); + using NodeType = NodeT; + using NodePtr = NodeT *; + using ParentPtr = decltype(std::declval<NodeT *>()->getParent()); + static_assert(std::is_pointer<ParentPtr>::value, + "Currently NodeT's parent must be a pointer type"); + using ParentType = typename std::remove_pointer<ParentPtr>::type; + static constexpr bool IsPostDominator = IsPostDom; + + using UpdateType = DomTreeBuilder::Update<NodePtr>; + using UpdateKind = DomTreeBuilder::UpdateKind; + static constexpr UpdateKind Insert = UpdateKind::Insert; + static constexpr UpdateKind Delete = UpdateKind::Delete; + protected: - std::vector<NodeT *> Roots; + // Dominators always have a single root, postdominators can have more. + SmallVector<NodeT *, IsPostDom ? 4 : 1> Roots; using DomTreeNodeMapType = DenseMap<NodeT *, std::unique_ptr<DomTreeNodeBase<NodeT>>>; DomTreeNodeMapType DomTreeNodes; DomTreeNodeBase<NodeT> *RootNode; - using ParentPtr = decltype(std::declval<NodeT *>()->getParent()); ParentPtr Parent = nullptr; mutable bool DFSInfoValid = false; @@ -224,12 +275,6 @@ class DominatorTreeBase { friend struct DomTreeBuilder::SemiNCAInfo<DominatorTreeBase>; public: - static_assert(std::is_pointer<typename GraphTraits<NodeT *>::NodeRef>::value, - "Currently DominatorTreeBase supports only pointer nodes"); - using NodeType = NodeT; - using NodePtr = NodeT *; - static constexpr bool IsPostDominator = IsPostDom; - DominatorTreeBase() {} DominatorTreeBase(DominatorTreeBase &&Arg) @@ -260,7 +305,7 @@ class DominatorTreeBase { /// multiple blocks if we are computing post dominators. For forward /// dominators, this will always be a single block (the entry node). /// - const std::vector<NodeT *> &getRoots() const { return Roots; } + const SmallVectorImpl<NodeT *> &getRoots() const { return Roots; } /// isPostDominator - Returns true if analysis based of postdoms /// @@ -412,14 +457,15 @@ class DominatorTreeBase { } /// findNearestCommonDominator - Find nearest common dominator basic block - /// for basic block A and B. If there is no such block then return NULL. + /// for basic block A and B. If there is no such block then return nullptr. NodeT *findNearestCommonDominator(NodeT *A, NodeT *B) const { + assert(A && B && "Pointers are not valid"); assert(A->getParent() == B->getParent() && "Two blocks are not in same function"); // If either A or B is a entry block then it is nearest common dominator // (for forward-dominators). - if (!this->isPostDominator()) { + if (!isPostDominator()) { NodeT &Entry = A->getParent()->front(); if (A == &Entry || B == &Entry) return &Entry; @@ -457,6 +503,41 @@ class DominatorTreeBase { // API to update (Post)DominatorTree information based on modifications to // the CFG... + /// Inform the dominator tree about a sequence of CFG edge insertions and + /// deletions and perform a batch update on the tree. + /// + /// This function should be used when there were multiple CFG updates after + /// the last dominator tree update. It takes care of performing the updates + /// in sync with the CFG and optimizes away the redundant operations that + /// cancel each other. + /// The functions expects the sequence of updates to be balanced. Eg.: + /// - {{Insert, A, B}, {Delete, A, B}, {Insert, A, B}} is fine, because + /// logically it results in a single insertions. + /// - {{Insert, A, B}, {Insert, A, B}} is invalid, because it doesn't make + /// sense to insert the same edge twice. + /// + /// What's more, the functions assumes that it's safe to ask every node in the + /// CFG about its children and inverse children. This implies that deletions + /// of CFG edges must not delete the CFG nodes before calling this function. + /// + /// Batch updates should be generally faster when performing longer sequences + /// of updates than calling insertEdge/deleteEdge manually multiple times, as + /// it can reorder the updates and remove redundant ones internally. + /// The batch updater is also able to detect sequences of zero and exactly one + /// update -- it's optimized to do less work in these cases. + /// + /// Note that for postdominators it automatically takes care of applying + /// updates on reverse edges internally (so there's no need to swap the + /// From and To pointers when constructing DominatorTree::UpdateType). + /// The type of updates is the same for DomTreeBase<T> and PostDomTreeBase<T> + /// with the same template parameter T. + /// + /// \param Updates An unordered sequence of updates to perform. + /// + void applyUpdates(ArrayRef<UpdateType> Updates) { + DomTreeBuilder::ApplyUpdates(*this, Updates); + } + /// Inform the dominator tree about a CFG edge insertion and update the tree. /// /// This function has to be called just before or just after making the update @@ -476,11 +557,10 @@ class DominatorTreeBase { /// Inform the dominator tree about a CFG edge deletion and update the tree. /// - /// This function has to be called just after making the update - /// on the actual CFG. There cannot be any other updates that the dominator - /// tree doesn't know about. The only exception is when the deletion that the - /// tree is informed about makes some (domominator) subtree unreachable -- in - /// this case, it is fine to perform deletions within this subtree. + /// This function has to be called just after making the update on the actual + /// CFG. An internal functions checks if the edge doesn't exist in the CFG in + /// DEBUG mode. There cannot be any other updates that the + /// dominator tree doesn't know about. /// /// Note that for postdominators it automatically takes care of deleting /// a reverse edge internally (so there's no need to swap the parameters). @@ -559,11 +639,12 @@ class DominatorTreeBase { assert(Node && "Removing node that isn't in dominator tree."); assert(Node->getChildren().empty() && "Node is not a leaf node."); + DFSInfoValid = false; + // Remove node from immediate dominator's children list. DomTreeNodeBase<NodeT> *IDom = Node->getIDom(); if (IDom) { - typename std::vector<DomTreeNodeBase<NodeT> *>::iterator I = - find(IDom->Children, Node); + const auto I = find(IDom->Children, Node); assert(I != IDom->Children.end() && "Not in immediate dominator children set!"); // I am no longer your child... @@ -571,6 +652,15 @@ class DominatorTreeBase { } DomTreeNodes.erase(BB); + + if (!IsPostDom) return; + + // Remember to update PostDominatorTree roots. + auto RIt = llvm::find(Roots, BB); + if (RIt != Roots.end()) { + std::swap(*RIt, Roots.back()); + Roots.pop_back(); + } } /// splitBlock - BB is split and now it has one successor. Update dominator @@ -586,7 +676,7 @@ class DominatorTreeBase { /// void print(raw_ostream &O) const { O << "=============================--------------------------------\n"; - if (this->isPostDominator()) + if (IsPostDominator) O << "Inorder PostDominator Tree: "; else O << "Inorder Dominator Tree: "; @@ -596,6 +686,14 @@ class DominatorTreeBase { // The postdom tree can have a null root if there are no returns. if (getRootNode()) PrintDomTree<NodeT>(getRootNode(), O, 1); + if (IsPostDominator) { + O << "Roots: "; + for (const NodePtr Block : Roots) { + Block->printAsOperand(O, false); + O << " "; + } + O << "\n"; + } } public: @@ -607,28 +705,25 @@ public: return; } - unsigned DFSNum = 0; - SmallVector<std::pair<const DomTreeNodeBase<NodeT> *, typename DomTreeNodeBase<NodeT>::const_iterator>, 32> WorkStack; const DomTreeNodeBase<NodeT> *ThisRoot = getRootNode(); - + assert((!Parent || ThisRoot) && "Empty constructed DomTree"); if (!ThisRoot) return; - // Even in the case of multiple exits that form the post dominator root - // nodes, do not iterate over all exits, but start from the virtual root - // node. Otherwise bbs, that are not post dominated by any exit but by the - // virtual root node, will never be assigned a DFS number. - WorkStack.push_back(std::make_pair(ThisRoot, ThisRoot->begin())); + // Both dominators and postdominators have a single root node. In the case + // case of PostDominatorTree, this node is a virtual root. + WorkStack.push_back({ThisRoot, ThisRoot->begin()}); + + unsigned DFSNum = 0; ThisRoot->DFSNumIn = DFSNum++; while (!WorkStack.empty()) { const DomTreeNodeBase<NodeT> *Node = WorkStack.back().first; - typename DomTreeNodeBase<NodeT>::const_iterator ChildIt = - WorkStack.back().second; + const auto ChildIt = WorkStack.back().second; // If we visited all of the children of this node, "recurse" back up the // stack setting the DFOutNum. @@ -640,7 +735,7 @@ public: const DomTreeNodeBase<NodeT> *Child = *ChildIt; ++WorkStack.back().second; - WorkStack.push_back(std::make_pair(Child, Child->begin())); + WorkStack.push_back({Child, Child->begin()}); Child->DFSNumIn = DFSNum++; } } @@ -650,23 +745,9 @@ public: } /// recalculate - compute a dominator tree for the given function - template <class FT> void recalculate(FT &F) { - using TraitsTy = GraphTraits<FT *>; - reset(); - Parent = &F; - - if (!IsPostDominator) { - // Initialize root - NodeT *entry = TraitsTy::getEntryNode(&F); - addRoot(entry); - } else { - // Initialize the roots list - for (auto *Node : nodes(&F)) - if (TraitsTy::child_begin(Node) == TraitsTy::child_end(Node)) - addRoot(Node); - } - - DomTreeBuilder::Calculate(*this, F); + void recalculate(ParentType &Func) { + Parent = &Func; + DomTreeBuilder::Calculate(*this); } /// verify - check parent and sibling property diff --git a/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h index be90afa4c3c8..8f801662d0fb 100644 --- a/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/contrib/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -21,8 +21,8 @@ /// faster than the almost-linear O(n*alpha(n)) version, even for large CFGs. /// /// The file uses the Depth Based Search algorithm to perform incremental -/// upates (insertion and deletions). The implemented algorithm is based on this -/// publication: +/// updates (insertion and deletions). The implemented algorithm is based on +/// this publication: /// /// An Experimental Study of Dynamic Dominators /// Loukas Georgiadis, et al., April 12 2016, pp. 5-7, 9-10: @@ -34,8 +34,10 @@ #define LLVM_SUPPORT_GENERICDOMTREECONSTRUCTION_H #include <queue> +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GenericDomTree.h" @@ -45,25 +47,12 @@ namespace llvm { namespace DomTreeBuilder { -template <typename NodePtr, bool Inverse> -struct ChildrenGetter { - static auto Get(NodePtr N) -> decltype(reverse(children<NodePtr>(N))) { - return reverse(children<NodePtr>(N)); - } -}; - -template <typename NodePtr> -struct ChildrenGetter<NodePtr, true> { - static auto Get(NodePtr N) -> decltype(inverse_children<NodePtr>(N)) { - return inverse_children<NodePtr>(N); - } -}; - template <typename DomTreeT> struct SemiNCAInfo { using NodePtr = typename DomTreeT::NodePtr; using NodeT = typename DomTreeT::NodeType; using TreeNodePtr = DomTreeNodeBase<NodeT> *; + using RootsT = decltype(DomTreeT::Roots); static constexpr bool IsPostDom = DomTreeT::IsPostDominator; // Information record used by Semi-NCA during tree construction. @@ -81,11 +70,99 @@ struct SemiNCAInfo { std::vector<NodePtr> NumToNode = {nullptr}; DenseMap<NodePtr, InfoRec> NodeToInfo; + using UpdateT = typename DomTreeT::UpdateType; + struct BatchUpdateInfo { + SmallVector<UpdateT, 4> Updates; + using NodePtrAndKind = PointerIntPair<NodePtr, 1, UpdateKind>; + + // In order to be able to walk a CFG that is out of sync with the CFG + // DominatorTree last knew about, use the list of updates to reconstruct + // previous CFG versions of the current CFG. For each node, we store a set + // of its virtually added/deleted future successors and predecessors. + // Note that these children are from the future relative to what the + // DominatorTree knows about -- using them to gets us some snapshot of the + // CFG from the past (relative to the state of the CFG). + DenseMap<NodePtr, SmallDenseSet<NodePtrAndKind, 4>> FutureSuccessors; + DenseMap<NodePtr, SmallDenseSet<NodePtrAndKind, 4>> FuturePredecessors; + // Remembers if the whole tree was recalculated at some point during the + // current batch update. + bool IsRecalculated = false; + }; + + BatchUpdateInfo *BatchUpdates; + using BatchUpdatePtr = BatchUpdateInfo *; + + // If BUI is a nullptr, then there's no batch update in progress. + SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {} + void clear() { NumToNode = {nullptr}; // Restore to initial state with a dummy start node. NodeToInfo.clear(); + // Don't reset the pointer to BatchUpdateInfo here -- if there's an update + // in progress, we need this information to continue it. } + template <bool Inverse> + struct ChildrenGetter { + using ResultTy = SmallVector<NodePtr, 8>; + + static ResultTy Get(NodePtr N, std::integral_constant<bool, false>) { + auto RChildren = reverse(children<NodePtr>(N)); + return ResultTy(RChildren.begin(), RChildren.end()); + } + + static ResultTy Get(NodePtr N, std::integral_constant<bool, true>) { + auto IChildren = inverse_children<NodePtr>(N); + return ResultTy(IChildren.begin(), IChildren.end()); + } + + using Tag = std::integral_constant<bool, Inverse>; + + // The function below is the core part of the batch updater. It allows the + // Depth Based Search algorithm to perform incremental updates in lockstep + // with updates to the CFG. We emulated lockstep CFG updates by getting its + // next snapshots by reverse-applying future updates. + static ResultTy Get(NodePtr N, BatchUpdatePtr BUI) { + ResultTy Res = Get(N, Tag()); + // If there's no batch update in progress, simply return node's children. + if (!BUI) return Res; + + // CFG children are actually its *most current* children, and we have to + // reverse-apply the future updates to get the node's children at the + // point in time the update was performed. + auto &FutureChildren = (Inverse != IsPostDom) ? BUI->FuturePredecessors + : BUI->FutureSuccessors; + auto FCIt = FutureChildren.find(N); + if (FCIt == FutureChildren.end()) return Res; + + for (auto ChildAndKind : FCIt->second) { + const NodePtr Child = ChildAndKind.getPointer(); + const UpdateKind UK = ChildAndKind.getInt(); + + // Reverse-apply the future update. + if (UK == UpdateKind::Insert) { + // If there's an insertion in the future, it means that the edge must + // exist in the current CFG, but was not present in it before. + assert(llvm::find(Res, Child) != Res.end() + && "Expected child not found in the CFG"); + Res.erase(std::remove(Res.begin(), Res.end(), Child), Res.end()); + DEBUG(dbgs() << "\tHiding edge " << BlockNamePrinter(N) << " -> " + << BlockNamePrinter(Child) << "\n"); + } else { + // If there's an deletion in the future, it means that the edge cannot + // exist in the current CFG, but existed in it before. + assert(llvm::find(Res, Child) == Res.end() && + "Unexpected child found in the CFG"); + DEBUG(dbgs() << "\tShowing virtual edge " << BlockNamePrinter(N) + << " -> " << BlockNamePrinter(Child) << "\n"); + Res.push_back(Child); + } + } + + return Res; + } + }; + NodePtr getIDom(NodePtr BB) const { auto InfoIt = NodeToInfo.find(BB); if (InfoIt == NodeToInfo.end()) return nullptr; @@ -131,7 +208,11 @@ struct SemiNCAInfo { // Custom DFS implementation which can skip nodes based on a provided // predicate. It also collects ReverseChildren so that we don't have to spend // time getting predecessors in SemiNCA. - template <bool Inverse, typename DescendCondition> + // + // If IsReverse is set to true, the DFS walk will be performed backwards + // relative to IsPostDom -- using reverse edges for dominators and forward + // edges for postdominators. + template <bool IsReverse = false, typename DescendCondition> unsigned runDFS(NodePtr V, unsigned LastNum, DescendCondition Condition, unsigned AttachToNum) { assert(V); @@ -148,10 +229,12 @@ struct SemiNCAInfo { BBInfo.Label = BB; NumToNode.push_back(BB); - for (const NodePtr Succ : ChildrenGetter<NodePtr, Inverse>::Get(BB)) { + constexpr bool Direction = IsReverse != IsPostDom; // XOR. + for (const NodePtr Succ : + ChildrenGetter<Direction>::Get(BB, BatchUpdates)) { const auto SIT = NodeToInfo.find(Succ); // Don't visit nodes more than once but remember to collect - // RerverseChildren. + // ReverseChildren. if (SIT != NodeToInfo.end() && SIT->second.DFSNum != 0) { if (Succ != BB) SIT->second.ReverseChildren.push_back(BB); continue; @@ -256,51 +339,244 @@ struct SemiNCAInfo { } } - template <typename DescendCondition> - unsigned doFullDFSWalk(const DomTreeT &DT, DescendCondition DC) { - unsigned Num = 0; + // PostDominatorTree always has a virtual root that represents a virtual CFG + // node that serves as a single exit from the function. All the other exits + // (CFG nodes with terminators and nodes in infinite loops are logically + // connected to this virtual CFG exit node). + // This functions maps a nullptr CFG node to the virtual root tree node. + void addVirtualRoot() { + assert(IsPostDom && "Only postdominators have a virtual root"); + assert(NumToNode.size() == 1 && "SNCAInfo must be freshly constructed"); - if (DT.Roots.size() > 1) { - auto &BBInfo = NodeToInfo[nullptr]; - BBInfo.DFSNum = BBInfo.Semi = ++Num; - BBInfo.Label = nullptr; + auto &BBInfo = NodeToInfo[nullptr]; + BBInfo.DFSNum = BBInfo.Semi = 1; + BBInfo.Label = nullptr; - NumToNode.push_back(nullptr); // NumToNode[n] = V; + NumToNode.push_back(nullptr); // NumToNode[1] = nullptr; + } + + // For postdominators, nodes with no forward successors are trivial roots that + // are always selected as tree roots. Roots with forward successors correspond + // to CFG nodes within infinite loops. + static bool HasForwardSuccessors(const NodePtr N, BatchUpdatePtr BUI) { + assert(N && "N must be a valid node"); + return !ChildrenGetter<false>::Get(N, BUI).empty(); + } + + static NodePtr GetEntryNode(const DomTreeT &DT) { + assert(DT.Parent && "Parent not set"); + return GraphTraits<typename DomTreeT::ParentPtr>::getEntryNode(DT.Parent); + } + + // Finds all roots without relaying on the set of roots already stored in the + // tree. + // We define roots to be some non-redundant set of the CFG nodes + static RootsT FindRoots(const DomTreeT &DT, BatchUpdatePtr BUI) { + assert(DT.Parent && "Parent pointer is not set"); + RootsT Roots; + + // For dominators, function entry CFG node is always a tree root node. + if (!IsPostDom) { + Roots.push_back(GetEntryNode(DT)); + return Roots; } - if (DT.isPostDominator()) { - for (auto *Root : DT.Roots) Num = runDFS<true>(Root, Num, DC, 1); - } else { - assert(DT.Roots.size() == 1); - Num = runDFS<false>(DT.Roots[0], Num, DC, Num); + SemiNCAInfo SNCA(BUI); + + // PostDominatorTree always has a virtual root. + SNCA.addVirtualRoot(); + unsigned Num = 1; + + DEBUG(dbgs() << "\t\tLooking for trivial roots\n"); + + // Step #1: Find all the trivial roots that are going to will definitely + // remain tree roots. + unsigned Total = 0; + // It may happen that there are some new nodes in the CFG that are result of + // the ongoing batch update, but we cannot really pretend that they don't + // exist -- we won't see any outgoing or incoming edges to them, so it's + // fine to discover them here, as they would end up appearing in the CFG at + // some point anyway. + for (const NodePtr N : nodes(DT.Parent)) { + ++Total; + // If it has no *successors*, it is definitely a root. + if (!HasForwardSuccessors(N, BUI)) { + Roots.push_back(N); + // Run DFS not to walk this part of CFG later. + Num = SNCA.runDFS(N, Num, AlwaysDescend, 1); + DEBUG(dbgs() << "Found a new trivial root: " << BlockNamePrinter(N) + << "\n"); + DEBUG(dbgs() << "Last visited node: " + << BlockNamePrinter(SNCA.NumToNode[Num]) << "\n"); + } } - return Num; + DEBUG(dbgs() << "\t\tLooking for non-trivial roots\n"); + + // Step #2: Find all non-trivial root candidates. Those are CFG nodes that + // are reverse-unreachable were not visited by previous DFS walks (i.e. CFG + // nodes in infinite loops). + bool HasNonTrivialRoots = false; + // Accounting for the virtual exit, see if we had any reverse-unreachable + // nodes. + if (Total + 1 != Num) { + HasNonTrivialRoots = true; + // Make another DFS pass over all other nodes to find the + // reverse-unreachable blocks, and find the furthest paths we'll be able + // to make. + // Note that this looks N^2, but it's really 2N worst case, if every node + // is unreachable. This is because we are still going to only visit each + // unreachable node once, we may just visit it in two directions, + // depending on how lucky we get. + SmallPtrSet<NodePtr, 4> ConnectToExitBlock; + for (const NodePtr I : nodes(DT.Parent)) { + if (SNCA.NodeToInfo.count(I) == 0) { + DEBUG(dbgs() << "\t\t\tVisiting node " << BlockNamePrinter(I) + << "\n"); + // Find the furthest away we can get by following successors, then + // follow them in reverse. This gives us some reasonable answer about + // the post-dom tree inside any infinite loop. In particular, it + // guarantees we get to the farthest away point along *some* + // path. This also matches the GCC's behavior. + // If we really wanted a totally complete picture of dominance inside + // this infinite loop, we could do it with SCC-like algorithms to find + // the lowest and highest points in the infinite loop. In theory, it + // would be nice to give the canonical backedge for the loop, but it's + // expensive and does not always lead to a minimal set of roots. + DEBUG(dbgs() << "\t\t\tRunning forward DFS\n"); + + const unsigned NewNum = SNCA.runDFS<true>(I, Num, AlwaysDescend, Num); + const NodePtr FurthestAway = SNCA.NumToNode[NewNum]; + DEBUG(dbgs() << "\t\t\tFound a new furthest away node " + << "(non-trivial root): " + << BlockNamePrinter(FurthestAway) << "\n"); + ConnectToExitBlock.insert(FurthestAway); + Roots.push_back(FurthestAway); + DEBUG(dbgs() << "\t\t\tPrev DFSNum: " << Num << ", new DFSNum: " + << NewNum << "\n\t\t\tRemoving DFS info\n"); + for (unsigned i = NewNum; i > Num; --i) { + const NodePtr N = SNCA.NumToNode[i]; + DEBUG(dbgs() << "\t\t\t\tRemoving DFS info for " + << BlockNamePrinter(N) << "\n"); + SNCA.NodeToInfo.erase(N); + SNCA.NumToNode.pop_back(); + } + const unsigned PrevNum = Num; + DEBUG(dbgs() << "\t\t\tRunning reverse DFS\n"); + Num = SNCA.runDFS(FurthestAway, Num, AlwaysDescend, 1); + for (unsigned i = PrevNum + 1; i <= Num; ++i) + DEBUG(dbgs() << "\t\t\t\tfound node " + << BlockNamePrinter(SNCA.NumToNode[i]) << "\n"); + } + } + } + + DEBUG(dbgs() << "Total: " << Total << ", Num: " << Num << "\n"); + DEBUG(dbgs() << "Discovered CFG nodes:\n"); + DEBUG(for (size_t i = 0; i <= Num; ++i) dbgs() + << i << ": " << BlockNamePrinter(SNCA.NumToNode[i]) << "\n"); + + assert((Total + 1 == Num) && "Everything should have been visited"); + + // Step #3: If we found some non-trivial roots, make them non-redundant. + if (HasNonTrivialRoots) RemoveRedundantRoots(DT, BUI, Roots); + + DEBUG(dbgs() << "Found roots: "); + DEBUG(for (auto *Root : Roots) dbgs() << BlockNamePrinter(Root) << " "); + DEBUG(dbgs() << "\n"); + + return Roots; } - void calculateFromScratch(DomTreeT &DT, const unsigned NumBlocks) { + // This function only makes sense for postdominators. + // We define roots to be some set of CFG nodes where (reverse) DFS walks have + // to start in order to visit all the CFG nodes (including the + // reverse-unreachable ones). + // When the search for non-trivial roots is done it may happen that some of + // the non-trivial roots are reverse-reachable from other non-trivial roots, + // which makes them redundant. This function removes them from the set of + // input roots. + static void RemoveRedundantRoots(const DomTreeT &DT, BatchUpdatePtr BUI, + RootsT &Roots) { + assert(IsPostDom && "This function is for postdominators only"); + DEBUG(dbgs() << "Removing redundant roots\n"); + + SemiNCAInfo SNCA(BUI); + + for (unsigned i = 0; i < Roots.size(); ++i) { + auto &Root = Roots[i]; + // Trivial roots are always non-redundant. + if (!HasForwardSuccessors(Root, BUI)) continue; + DEBUG(dbgs() << "\tChecking if " << BlockNamePrinter(Root) + << " remains a root\n"); + SNCA.clear(); + // Do a forward walk looking for the other roots. + const unsigned Num = SNCA.runDFS<true>(Root, 0, AlwaysDescend, 0); + // Skip the start node and begin from the second one (note that DFS uses + // 1-based indexing). + for (unsigned x = 2; x <= Num; ++x) { + const NodePtr N = SNCA.NumToNode[x]; + // If we wound another root in a (forward) DFS walk, remove the current + // root from the set of roots, as it is reverse-reachable from the other + // one. + if (llvm::find(Roots, N) != Roots.end()) { + DEBUG(dbgs() << "\tForward DFS walk found another root " + << BlockNamePrinter(N) << "\n\tRemoving root " + << BlockNamePrinter(Root) << "\n"); + std::swap(Root, Roots.back()); + Roots.pop_back(); + + // Root at the back takes the current root's place. + // Start the next loop iteration with the same index. + --i; + break; + } + } + } + } + + template <typename DescendCondition> + void doFullDFSWalk(const DomTreeT &DT, DescendCondition DC) { + if (!IsPostDom) { + assert(DT.Roots.size() == 1 && "Dominators should have a singe root"); + runDFS(DT.Roots[0], 0, DC, 0); + return; + } + + addVirtualRoot(); + unsigned Num = 1; + for (const NodePtr Root : DT.Roots) Num = runDFS(Root, Num, DC, 0); + } + + static void CalculateFromScratch(DomTreeT &DT, BatchUpdatePtr BUI) { + auto *Parent = DT.Parent; + DT.reset(); + DT.Parent = Parent; + SemiNCAInfo SNCA(nullptr); // Since we are rebuilding the whole tree, + // there's no point doing it incrementally. + // Step #0: Number blocks in depth-first order and initialize variables used // in later stages of the algorithm. - const unsigned LastDFSNum = doFullDFSWalk(DT, AlwaysDescend); + DT.Roots = FindRoots(DT, nullptr); + SNCA.doFullDFSWalk(DT, AlwaysDescend); - runSemiNCA(DT); + SNCA.runSemiNCA(DT); + if (BUI) { + BUI->IsRecalculated = true; + DEBUG(dbgs() << "DomTree recalculated, skipping future batch updates\n"); + } if (DT.Roots.empty()) return; - // Add a node for the root. This node might be the actual root, if there is - // one exit block, or it may be the virtual exit (denoted by - // (BasicBlock *)0) which postdominates all real exits if there are multiple - // exit blocks, or an infinite loop. - // It might be that some blocks did not get a DFS number (e.g., blocks of - // infinite loops). In these cases an artificial exit node is required. - const bool MultipleRoots = DT.Roots.size() > 1 || (DT.isPostDominator() && - LastDFSNum != NumBlocks); - NodePtr Root = !MultipleRoots ? DT.Roots[0] : nullptr; + // Add a node for the root. If the tree is a PostDominatorTree it will be + // the virtual exit (denoted by (BasicBlock *) nullptr) which postdominates + // all real exits (including multiple exit blocks, infinite loops). + NodePtr Root = IsPostDom ? nullptr : DT.Roots[0]; DT.RootNode = (DT.DomTreeNodes[Root] = llvm::make_unique<DomTreeNodeBase<NodeT>>(Root, nullptr)) .get(); - attachNewSubtree(DT, DT.RootNode); + SNCA.attachNewSubtree(DT, DT.RootNode); } void attachNewSubtree(DomTreeT& DT, const TreeNodePtr AttachTo) { @@ -317,11 +593,11 @@ struct SemiNCAInfo { NodePtr ImmDom = getIDom(W); - // Get or calculate the node for the immediate dominator + // Get or calculate the node for the immediate dominator. TreeNodePtr IDomNode = getNodeForBlock(ImmDom, DT); // Add a new tree node for this BasicBlock, and link it as a child of - // IDomNode + // IDomNode. DT.DomTreeNodes[W] = IDomNode->addChild( llvm::make_unique<DomTreeNodeBase<NodeT>>(W, IDomNode)); } @@ -357,31 +633,105 @@ struct SemiNCAInfo { SmallVector<TreeNodePtr, 8> VisitedNotAffectedQueue; }; - static void InsertEdge(DomTreeT &DT, const NodePtr From, const NodePtr To) { - assert(From && To && "Cannot connect nullptrs"); + static void InsertEdge(DomTreeT &DT, const BatchUpdatePtr BUI, + const NodePtr From, const NodePtr To) { + assert((From || IsPostDom) && + "From has to be a valid CFG node or a virtual root"); + assert(To && "Cannot be a nullptr"); DEBUG(dbgs() << "Inserting edge " << BlockNamePrinter(From) << " -> " << BlockNamePrinter(To) << "\n"); - const TreeNodePtr FromTN = DT.getNode(From); - - // Ignore edges from unreachable nodes. - if (!FromTN) return; + TreeNodePtr FromTN = DT.getNode(From); + + if (!FromTN) { + // Ignore edges from unreachable nodes for (forward) dominators. + if (!IsPostDom) return; + + // The unreachable node becomes a new root -- a tree node for it. + TreeNodePtr VirtualRoot = DT.getNode(nullptr); + FromTN = + (DT.DomTreeNodes[From] = VirtualRoot->addChild( + llvm::make_unique<DomTreeNodeBase<NodeT>>(From, VirtualRoot))) + .get(); + DT.Roots.push_back(From); + } DT.DFSInfoValid = false; const TreeNodePtr ToTN = DT.getNode(To); if (!ToTN) - InsertUnreachable(DT, FromTN, To); + InsertUnreachable(DT, BUI, FromTN, To); else - InsertReachable(DT, FromTN, ToTN); + InsertReachable(DT, BUI, FromTN, ToTN); + } + + // Determines if some existing root becomes reverse-reachable after the + // insertion. Rebuilds the whole tree if that situation happens. + static bool UpdateRootsBeforeInsertion(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr From, + const TreeNodePtr To) { + assert(IsPostDom && "This function is only for postdominators"); + // Destination node is not attached to the virtual root, so it cannot be a + // root. + if (!DT.isVirtualRoot(To->getIDom())) return false; + + auto RIt = llvm::find(DT.Roots, To->getBlock()); + if (RIt == DT.Roots.end()) + return false; // To is not a root, nothing to update. + + DEBUG(dbgs() << "\t\tAfter the insertion, " << BlockNamePrinter(To) + << " is no longer a root\n\t\tRebuilding the tree!!!\n"); + + CalculateFromScratch(DT, BUI); + return true; + } + + // Updates the set of roots after insertion or deletion. This ensures that + // roots are the same when after a series of updates and when the tree would + // be built from scratch. + static void UpdateRootsAfterUpdate(DomTreeT &DT, const BatchUpdatePtr BUI) { + assert(IsPostDom && "This function is only for postdominators"); + + // The tree has only trivial roots -- nothing to update. + if (std::none_of(DT.Roots.begin(), DT.Roots.end(), [BUI](const NodePtr N) { + return HasForwardSuccessors(N, BUI); + })) + return; + + // Recalculate the set of roots. + DT.Roots = FindRoots(DT, BUI); + for (const NodePtr R : DT.Roots) { + const TreeNodePtr TN = DT.getNode(R); + // A CFG node was selected as a tree root, but the corresponding tree node + // is not connected to the virtual root. This is because the incremental + // algorithm does not really know or use the set of roots and can make a + // different (implicit) decision about which nodes within an infinite loop + // becomes a root. + if (DT.isVirtualRoot(TN->getIDom())) { + DEBUG(dbgs() << "Root " << BlockNamePrinter(R) + << " is not virtual root's child\n" + << "The entire tree needs to be rebuilt\n"); + // It should be possible to rotate the subtree instead of recalculating + // the whole tree, but this situation happens extremely rarely in + // practice. + CalculateFromScratch(DT, BUI); + return; + } + } } // Handles insertion to a node already in the dominator tree. - static void InsertReachable(DomTreeT &DT, const TreeNodePtr From, - const TreeNodePtr To) { + static void InsertReachable(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr From, const TreeNodePtr To) { DEBUG(dbgs() << "\tReachable " << BlockNamePrinter(From->getBlock()) << " -> " << BlockNamePrinter(To->getBlock()) << "\n"); + if (IsPostDom && UpdateRootsBeforeInsertion(DT, BUI, From, To)) return; + // DT.findNCD expects both pointers to be valid. When From is a virtual + // root, then its CFG block pointer is a nullptr, so we have to 'compute' + // the NCD manually. const NodePtr NCDBlock = - DT.findNearestCommonDominator(From->getBlock(), To->getBlock()); + (From->getBlock() && To->getBlock()) + ? DT.findNearestCommonDominator(From->getBlock(), To->getBlock()) + : nullptr; assert(NCDBlock || DT.isPostDominator()); const TreeNodePtr NCD = DT.getNode(NCDBlock); assert(NCD); @@ -410,53 +760,61 @@ struct SemiNCAInfo { II.AffectedQueue.push_back(CurrentNode); // Discover and collect affected successors of the current node. - VisitInsertion(DT, CurrentNode, CurrentNode->getLevel(), NCD, II); + VisitInsertion(DT, BUI, CurrentNode, CurrentNode->getLevel(), NCD, II); } // Finish by updating immediate dominators and levels. - UpdateInsertion(DT, NCD, II); + UpdateInsertion(DT, BUI, NCD, II); } // Visits an affected node and collect its affected successors. - static void VisitInsertion(DomTreeT &DT, const TreeNodePtr TN, - const unsigned RootLevel, const TreeNodePtr NCD, - InsertionInfo &II) { + static void VisitInsertion(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr TN, const unsigned RootLevel, + const TreeNodePtr NCD, InsertionInfo &II) { const unsigned NCDLevel = NCD->getLevel(); DEBUG(dbgs() << "Visiting " << BlockNamePrinter(TN) << "\n"); - assert(TN->getBlock()); - for (const NodePtr Succ : - ChildrenGetter<NodePtr, IsPostDom>::Get(TN->getBlock())) { - const TreeNodePtr SuccTN = DT.getNode(Succ); - assert(SuccTN && "Unreachable successor found at reachable insertion"); - const unsigned SuccLevel = SuccTN->getLevel(); - - DEBUG(dbgs() << "\tSuccessor " << BlockNamePrinter(Succ) - << ", level = " << SuccLevel << "\n"); - - // Succ dominated by subtree From -- not affected. - // (Based on the lemma 2.5 from the second paper.) - if (SuccLevel > RootLevel) { - DEBUG(dbgs() << "\t\tDominated by subtree From\n"); - if (II.Visited.count(SuccTN) != 0) continue; - - DEBUG(dbgs() << "\t\tMarking visited not affected " - << BlockNamePrinter(Succ) << "\n"); - II.Visited.insert(SuccTN); - II.VisitedNotAffectedQueue.push_back(SuccTN); - VisitInsertion(DT, SuccTN, RootLevel, NCD, II); - } else if ((SuccLevel > NCDLevel + 1) && II.Affected.count(SuccTN) == 0) { - DEBUG(dbgs() << "\t\tMarking affected and adding " - << BlockNamePrinter(Succ) << " to a Bucket\n"); - II.Affected.insert(SuccTN); - II.Bucket.push({SuccLevel, SuccTN}); + SmallVector<TreeNodePtr, 8> Stack = {TN}; + assert(TN->getBlock() && II.Visited.count(TN) && "Preconditions!"); + + do { + TreeNodePtr Next = Stack.pop_back_val(); + + for (const NodePtr Succ : + ChildrenGetter<IsPostDom>::Get(Next->getBlock(), BUI)) { + const TreeNodePtr SuccTN = DT.getNode(Succ); + assert(SuccTN && "Unreachable successor found at reachable insertion"); + const unsigned SuccLevel = SuccTN->getLevel(); + + DEBUG(dbgs() << "\tSuccessor " << BlockNamePrinter(Succ) + << ", level = " << SuccLevel << "\n"); + + // Succ dominated by subtree From -- not affected. + // (Based on the lemma 2.5 from the second paper.) + if (SuccLevel > RootLevel) { + DEBUG(dbgs() << "\t\tDominated by subtree From\n"); + if (II.Visited.count(SuccTN) != 0) + continue; + + DEBUG(dbgs() << "\t\tMarking visited not affected " + << BlockNamePrinter(Succ) << "\n"); + II.Visited.insert(SuccTN); + II.VisitedNotAffectedQueue.push_back(SuccTN); + Stack.push_back(SuccTN); + } else if ((SuccLevel > NCDLevel + 1) && + II.Affected.count(SuccTN) == 0) { + DEBUG(dbgs() << "\t\tMarking affected and adding " + << BlockNamePrinter(Succ) << " to a Bucket\n"); + II.Affected.insert(SuccTN); + II.Bucket.push({SuccLevel, SuccTN}); + } } - } + } while (!Stack.empty()); } // Updates immediate dominators and levels after insertion. - static void UpdateInsertion(DomTreeT &DT, const TreeNodePtr NCD, - InsertionInfo &II) { + static void UpdateInsertion(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr NCD, InsertionInfo &II) { DEBUG(dbgs() << "Updating NCD = " << BlockNamePrinter(NCD) << "\n"); for (const TreeNodePtr TN : II.AffectedQueue) { @@ -466,6 +824,7 @@ struct SemiNCAInfo { } UpdateLevelsAfterInsertion(II); + if (IsPostDom) UpdateRootsAfterUpdate(DT, BUI); } static void UpdateLevelsAfterInsertion(InsertionInfo &II) { @@ -480,36 +839,35 @@ struct SemiNCAInfo { } // Handles insertion to previously unreachable nodes. - static void InsertUnreachable(DomTreeT &DT, const TreeNodePtr From, - const NodePtr To) { + static void InsertUnreachable(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr From, const NodePtr To) { DEBUG(dbgs() << "Inserting " << BlockNamePrinter(From) << " -> (unreachable) " << BlockNamePrinter(To) << "\n"); // Collect discovered edges to already reachable nodes. SmallVector<std::pair<NodePtr, TreeNodePtr>, 8> DiscoveredEdgesToReachable; // Discover and connect nodes that became reachable with the insertion. - ComputeUnreachableDominators(DT, To, From, DiscoveredEdgesToReachable); + ComputeUnreachableDominators(DT, BUI, To, From, DiscoveredEdgesToReachable); DEBUG(dbgs() << "Inserted " << BlockNamePrinter(From) << " -> (prev unreachable) " << BlockNamePrinter(To) << "\n"); - DEBUG(DT.print(dbgs())); - // Used the discovered edges and inset discovered connecting (incoming) // edges. for (const auto &Edge : DiscoveredEdgesToReachable) { DEBUG(dbgs() << "\tInserting discovered connecting edge " << BlockNamePrinter(Edge.first) << " -> " << BlockNamePrinter(Edge.second) << "\n"); - InsertReachable(DT, DT.getNode(Edge.first), Edge.second); + InsertReachable(DT, BUI, DT.getNode(Edge.first), Edge.second); } } // Connects nodes that become reachable with an insertion. static void ComputeUnreachableDominators( - DomTreeT &DT, const NodePtr Root, const TreeNodePtr Incoming, + DomTreeT &DT, const BatchUpdatePtr BUI, const NodePtr Root, + const TreeNodePtr Incoming, SmallVectorImpl<std::pair<NodePtr, TreeNodePtr>> - &DiscoveredConnectingEdges) { + &DiscoveredConnectingEdges) { assert(!DT.getNode(Root) && "Root must not be reachable"); // Visit only previously unreachable nodes. @@ -522,50 +880,16 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA; - SNCA.runDFS<IsPostDom>(Root, 0, UnreachableDescender, 0); + SemiNCAInfo SNCA(BUI); + SNCA.runDFS(Root, 0, UnreachableDescender, 0); SNCA.runSemiNCA(DT); SNCA.attachNewSubtree(DT, Incoming); DEBUG(dbgs() << "After adding unreachable nodes\n"); - DEBUG(DT.print(dbgs())); } - // Checks if the tree contains all reachable nodes in the input graph. - bool verifyReachability(const DomTreeT &DT) { - clear(); - doFullDFSWalk(DT, AlwaysDescend); - - for (auto &NodeToTN : DT.DomTreeNodes) { - const TreeNodePtr TN = NodeToTN.second.get(); - const NodePtr BB = TN->getBlock(); - - // Virtual root has a corresponding virtual CFG node. - if (DT.isVirtualRoot(TN)) continue; - - if (NodeToInfo.count(BB) == 0) { - errs() << "DomTree node " << BlockNamePrinter(BB) - << " not found by DFS walk!\n"; - errs().flush(); - - return false; - } - } - - for (const NodePtr N : NumToNode) { - if (N && !DT.getNode(N)) { - errs() << "CFG node " << BlockNamePrinter(N) - << " not found in the DomTree!\n"; - errs().flush(); - - return false; - } - } - - return true; - } - - static void DeleteEdge(DomTreeT &DT, const NodePtr From, const NodePtr To) { + static void DeleteEdge(DomTreeT &DT, const BatchUpdatePtr BUI, + const NodePtr From, const NodePtr To) { assert(From && To && "Cannot disconnect nullptrs"); DEBUG(dbgs() << "Deleting edge " << BlockNamePrinter(From) << " -> " << BlockNamePrinter(To) << "\n"); @@ -574,8 +898,8 @@ struct SemiNCAInfo { // Ensure that the edge was in fact deleted from the CFG before informing // the DomTree about it. // The check is O(N), so run it only in debug configuration. - auto IsSuccessor = [](const NodePtr SuccCandidate, const NodePtr Of) { - auto Successors = ChildrenGetter<NodePtr, IsPostDom>::Get(Of); + auto IsSuccessor = [BUI](const NodePtr SuccCandidate, const NodePtr Of) { + auto Successors = ChildrenGetter<IsPostDom>::Get(Of, BUI); return llvm::find(Successors, SuccCandidate) != Successors.end(); }; (void)IsSuccessor; @@ -587,27 +911,37 @@ struct SemiNCAInfo { if (!FromTN) return; const TreeNodePtr ToTN = DT.getNode(To); - assert(ToTN && "To already unreachable -- there is no edge to delete"); + if (!ToTN) { + DEBUG(dbgs() << "\tTo (" << BlockNamePrinter(To) + << ") already unreachable -- there is no edge to delete\n"); + return; + } + const NodePtr NCDBlock = DT.findNearestCommonDominator(From, To); const TreeNodePtr NCD = DT.getNode(NCDBlock); // To dominates From -- nothing to do. if (ToTN == NCD) return; + DT.DFSInfoValid = false; + const TreeNodePtr ToIDom = ToTN->getIDom(); DEBUG(dbgs() << "\tNCD " << BlockNamePrinter(NCD) << ", ToIDom " << BlockNamePrinter(ToIDom) << "\n"); // To remains reachable after deletion. // (Based on the caption under Figure 4. from the second paper.) - if (FromTN != ToIDom || HasProperSupport(DT, ToTN)) - DeleteReachable(DT, FromTN, ToTN); + if (FromTN != ToIDom || HasProperSupport(DT, BUI, ToTN)) + DeleteReachable(DT, BUI, FromTN, ToTN); else - DeleteUnreachable(DT, ToTN); + DeleteUnreachable(DT, BUI, ToTN); + + if (IsPostDom) UpdateRootsAfterUpdate(DT, BUI); } // Handles deletions that leave destination nodes reachable. - static void DeleteReachable(DomTreeT &DT, const TreeNodePtr FromTN, + static void DeleteReachable(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr FromTN, const TreeNodePtr ToTN) { DEBUG(dbgs() << "Deleting reachable " << BlockNamePrinter(FromTN) << " -> " << BlockNamePrinter(ToTN) << "\n"); @@ -625,7 +959,7 @@ struct SemiNCAInfo { // scratch. if (!PrevIDomSubTree) { DEBUG(dbgs() << "The entire tree needs to be rebuilt\n"); - DT.recalculate(*DT.Parent); + CalculateFromScratch(DT, BUI); return; } @@ -637,8 +971,8 @@ struct SemiNCAInfo { DEBUG(dbgs() << "\tTop of subtree: " << BlockNamePrinter(ToIDomTN) << "\n"); - SemiNCAInfo SNCA; - SNCA.runDFS<IsPostDom>(ToIDom, 0, DescendBelow, 0); + SemiNCAInfo SNCA(BUI); + SNCA.runDFS(ToIDom, 0, DescendBelow, 0); DEBUG(dbgs() << "\tRunning Semi-NCA\n"); SNCA.runSemiNCA(DT, Level); SNCA.reattachExistingSubtree(DT, PrevIDomSubTree); @@ -646,10 +980,11 @@ struct SemiNCAInfo { // Checks if a node has proper support, as defined on the page 3 and later // explained on the page 7 of the second paper. - static bool HasProperSupport(DomTreeT &DT, const TreeNodePtr TN) { + static bool HasProperSupport(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr TN) { DEBUG(dbgs() << "IsReachableFromIDom " << BlockNamePrinter(TN) << "\n"); for (const NodePtr Pred : - ChildrenGetter<NodePtr, !IsPostDom>::Get(TN->getBlock())) { + ChildrenGetter<!IsPostDom>::Get(TN->getBlock(), BUI)) { DEBUG(dbgs() << "\tPred " << BlockNamePrinter(Pred) << "\n"); if (!DT.getNode(Pred)) continue; @@ -669,12 +1004,24 @@ struct SemiNCAInfo { // Handle deletions that make destination node unreachable. // (Based on the lemma 2.7 from the second paper.) - static void DeleteUnreachable(DomTreeT &DT, const TreeNodePtr ToTN) { + static void DeleteUnreachable(DomTreeT &DT, const BatchUpdatePtr BUI, + const TreeNodePtr ToTN) { DEBUG(dbgs() << "Deleting unreachable subtree " << BlockNamePrinter(ToTN) << "\n"); assert(ToTN); assert(ToTN->getBlock()); + if (IsPostDom) { + // Deletion makes a region reverse-unreachable and creates a new root. + // Simulate that by inserting an edge from the virtual root to ToTN and + // adding it as a new root. + DEBUG(dbgs() << "\tDeletion made a region reverse-unreachable\n"); + DEBUG(dbgs() << "\tAdding new root " << BlockNamePrinter(ToTN) << "\n"); + DT.Roots.push_back(ToTN->getBlock()); + InsertReachable(DT, BUI, DT.getNode(nullptr), ToTN); + return; + } + SmallVector<NodePtr, 16> AffectedQueue; const unsigned Level = ToTN->getLevel(); @@ -690,13 +1037,13 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA; + SemiNCAInfo SNCA(BUI); unsigned LastDFSNum = - SNCA.runDFS<IsPostDom>(ToTN->getBlock(), 0, DescendAndCollect, 0); + SNCA.runDFS(ToTN->getBlock(), 0, DescendAndCollect, 0); TreeNodePtr MinNode = ToTN; - // Identify the top of the subtree to rebuilt by finding the NCD of all + // Identify the top of the subtree to rebuild by finding the NCD of all // the affected nodes. for (const NodePtr N : AffectedQueue) { const TreeNodePtr TN = DT.getNode(N); @@ -715,7 +1062,7 @@ struct SemiNCAInfo { // Root reached, rebuild the whole tree from scratch. if (!MinNode->getIDom()) { DEBUG(dbgs() << "The entire tree needs to be rebuilt\n"); - DT.recalculate(*DT.Parent); + CalculateFromScratch(DT, BUI); return; } @@ -744,7 +1091,7 @@ struct SemiNCAInfo { const TreeNodePtr ToTN = DT.getNode(To); return ToTN && ToTN->getLevel() > MinLevel; }; - SNCA.runDFS<IsPostDom>(MinNode->getBlock(), 0, DescendBelow, 0); + SNCA.runDFS(MinNode->getBlock(), 0, DescendBelow, 0); DEBUG(dbgs() << "Previous IDom(MinNode) = " << BlockNamePrinter(PrevIDom) << "\nRunning Semi-NCA\n"); @@ -771,9 +1118,222 @@ struct SemiNCAInfo { } //~~ + //===--------------------- DomTree Batch Updater --------------------------=== + //~~ + + static void ApplyUpdates(DomTreeT &DT, ArrayRef<UpdateT> Updates) { + const size_t NumUpdates = Updates.size(); + if (NumUpdates == 0) + return; + + // Take the fast path for a single update and avoid running the batch update + // machinery. + if (NumUpdates == 1) { + const auto &Update = Updates.front(); + if (Update.getKind() == UpdateKind::Insert) + DT.insertEdge(Update.getFrom(), Update.getTo()); + else + DT.deleteEdge(Update.getFrom(), Update.getTo()); + + return; + } + + BatchUpdateInfo BUI; + LegalizeUpdates(Updates, BUI.Updates); + + const size_t NumLegalized = BUI.Updates.size(); + BUI.FutureSuccessors.reserve(NumLegalized); + BUI.FuturePredecessors.reserve(NumLegalized); + + // Use the legalized future updates to initialize future successors and + // predecessors. Note that these sets will only decrease size over time, as + // the next CFG snapshots slowly approach the actual (current) CFG. + for (UpdateT &U : BUI.Updates) { + BUI.FutureSuccessors[U.getFrom()].insert({U.getTo(), U.getKind()}); + BUI.FuturePredecessors[U.getTo()].insert({U.getFrom(), U.getKind()}); + } + + DEBUG(dbgs() << "About to apply " << NumLegalized << " updates\n"); + DEBUG(if (NumLegalized < 32) for (const auto &U + : reverse(BUI.Updates)) dbgs() + << '\t' << U << "\n"); + DEBUG(dbgs() << "\n"); + + // If the DominatorTree was recalculated at some point, stop the batch + // updates. Full recalculations ignore batch updates and look at the actual + // CFG. + for (size_t i = 0; i < NumLegalized && !BUI.IsRecalculated; ++i) + ApplyNextUpdate(DT, BUI); + } + + // This function serves double purpose: + // a) It removes redundant updates, which makes it easier to reverse-apply + // them when traversing CFG. + // b) It optimizes away updates that cancel each other out, as the end result + // is the same. + // + // It relies on the property of the incremental updates that says that the + // order of updates doesn't matter. This allows us to reorder them and end up + // with the exact same DomTree every time. + // + // Following the same logic, the function doesn't care about the order of + // input updates, so it's OK to pass it an unordered sequence of updates, that + // doesn't make sense when applied sequentially, eg. performing double + // insertions or deletions and then doing an opposite update. + // + // In the future, it should be possible to schedule updates in way that + // minimizes the amount of work needed done during incremental updates. + static void LegalizeUpdates(ArrayRef<UpdateT> AllUpdates, + SmallVectorImpl<UpdateT> &Result) { + DEBUG(dbgs() << "Legalizing " << AllUpdates.size() << " updates\n"); + // Count the total number of inserions of each edge. + // Each insertion adds 1 and deletion subtracts 1. The end number should be + // one of {-1 (deletion), 0 (NOP), +1 (insertion)}. Otherwise, the sequence + // of updates contains multiple updates of the same kind and we assert for + // that case. + SmallDenseMap<std::pair<NodePtr, NodePtr>, int, 4> Operations; + Operations.reserve(AllUpdates.size()); + + for (const auto &U : AllUpdates) { + NodePtr From = U.getFrom(); + NodePtr To = U.getTo(); + if (IsPostDom) std::swap(From, To); // Reverse edge for postdominators. + + Operations[{From, To}] += (U.getKind() == UpdateKind::Insert ? 1 : -1); + } + + Result.clear(); + Result.reserve(Operations.size()); + for (auto &Op : Operations) { + const int NumInsertions = Op.second; + assert(std::abs(NumInsertions) <= 1 && "Unbalanced operations!"); + if (NumInsertions == 0) continue; + const UpdateKind UK = + NumInsertions > 0 ? UpdateKind::Insert : UpdateKind::Delete; + Result.push_back({UK, Op.first.first, Op.first.second}); + } + + // Make the order consistent by not relying on pointer values within the + // set. Reuse the old Operations map. + // In the future, we should sort by something else to minimize the amount + // of work needed to perform the series of updates. + for (size_t i = 0, e = AllUpdates.size(); i != e; ++i) { + const auto &U = AllUpdates[i]; + if (!IsPostDom) + Operations[{U.getFrom(), U.getTo()}] = int(i); + else + Operations[{U.getTo(), U.getFrom()}] = int(i); + } + + std::sort(Result.begin(), Result.end(), + [&Operations](const UpdateT &A, const UpdateT &B) { + return Operations[{A.getFrom(), A.getTo()}] > + Operations[{B.getFrom(), B.getTo()}]; + }); + } + + static void ApplyNextUpdate(DomTreeT &DT, BatchUpdateInfo &BUI) { + assert(!BUI.Updates.empty() && "No updates to apply!"); + UpdateT CurrentUpdate = BUI.Updates.pop_back_val(); + DEBUG(dbgs() << "Applying update: " << CurrentUpdate << "\n"); + + // Move to the next snapshot of the CFG by removing the reverse-applied + // current update. + auto &FS = BUI.FutureSuccessors[CurrentUpdate.getFrom()]; + FS.erase({CurrentUpdate.getTo(), CurrentUpdate.getKind()}); + if (FS.empty()) BUI.FutureSuccessors.erase(CurrentUpdate.getFrom()); + + auto &FP = BUI.FuturePredecessors[CurrentUpdate.getTo()]; + FP.erase({CurrentUpdate.getFrom(), CurrentUpdate.getKind()}); + if (FP.empty()) BUI.FuturePredecessors.erase(CurrentUpdate.getTo()); + + if (CurrentUpdate.getKind() == UpdateKind::Insert) + InsertEdge(DT, &BUI, CurrentUpdate.getFrom(), CurrentUpdate.getTo()); + else + DeleteEdge(DT, &BUI, CurrentUpdate.getFrom(), CurrentUpdate.getTo()); + } + + //~~ //===--------------- DomTree correctness verification ---------------------=== //~~ + // Check if the tree has correct roots. A DominatorTree always has a single + // root which is the function's entry node. A PostDominatorTree can have + // multiple roots - one for each node with no successors and for infinite + // loops. + bool verifyRoots(const DomTreeT &DT) { + if (!DT.Parent && !DT.Roots.empty()) { + errs() << "Tree has no parent but has roots!\n"; + errs().flush(); + return false; + } + + if (!IsPostDom) { + if (DT.Roots.empty()) { + errs() << "Tree doesn't have a root!\n"; + errs().flush(); + return false; + } + + if (DT.getRoot() != GetEntryNode(DT)) { + errs() << "Tree's root is not its parent's entry node!\n"; + errs().flush(); + return false; + } + } + + RootsT ComputedRoots = FindRoots(DT, nullptr); + if (DT.Roots.size() != ComputedRoots.size() || + !std::is_permutation(DT.Roots.begin(), DT.Roots.end(), + ComputedRoots.begin())) { + errs() << "Tree has different roots than freshly computed ones!\n"; + errs() << "\tPDT roots: "; + for (const NodePtr N : DT.Roots) errs() << BlockNamePrinter(N) << ", "; + errs() << "\n\tComputed roots: "; + for (const NodePtr N : ComputedRoots) + errs() << BlockNamePrinter(N) << ", "; + errs() << "\n"; + errs().flush(); + return false; + } + + return true; + } + + // Checks if the tree contains all reachable nodes in the input graph. + bool verifyReachability(const DomTreeT &DT) { + clear(); + doFullDFSWalk(DT, AlwaysDescend); + + for (auto &NodeToTN : DT.DomTreeNodes) { + const TreeNodePtr TN = NodeToTN.second.get(); + const NodePtr BB = TN->getBlock(); + + // Virtual root has a corresponding virtual CFG node. + if (DT.isVirtualRoot(TN)) continue; + + if (NodeToInfo.count(BB) == 0) { + errs() << "DomTree node " << BlockNamePrinter(BB) + << " not found by DFS walk!\n"; + errs().flush(); + + return false; + } + } + + for (const NodePtr N : NumToNode) { + if (N && !DT.getNode(N)) { + errs() << "CFG node " << BlockNamePrinter(N) + << " not found in the DomTree!\n"; + errs().flush(); + + return false; + } + } + + return true; + } + // Check if for every parent with a level L in the tree all of its children // have level L + 1. static bool VerifyLevels(const DomTreeT &DT) { @@ -805,35 +1365,97 @@ struct SemiNCAInfo { return true; } - // Checks if for every edge From -> To in the graph - // NCD(From, To) == IDom(To) or To. - bool verifyNCD(const DomTreeT &DT) { - clear(); - doFullDFSWalk(DT, AlwaysDescend); + // Check if the computed DFS numbers are correct. Note that DFS info may not + // be valid, and when that is the case, we don't verify the numbers. + static bool VerifyDFSNumbers(const DomTreeT &DT) { + if (!DT.DFSInfoValid || !DT.Parent) + return true; - for (auto &BlockToInfo : NodeToInfo) { - auto &Info = BlockToInfo.second; + const NodePtr RootBB = IsPostDom ? nullptr : DT.getRoots()[0]; + const TreeNodePtr Root = DT.getNode(RootBB); - const NodePtr From = NumToNode[Info.Parent]; - if (!From) continue; + auto PrintNodeAndDFSNums = [](const TreeNodePtr TN) { + errs() << BlockNamePrinter(TN) << " {" << TN->getDFSNumIn() << ", " + << TN->getDFSNumOut() << '}'; + }; - const NodePtr To = BlockToInfo.first; - const TreeNodePtr ToTN = DT.getNode(To); - assert(ToTN); - - const NodePtr NCD = DT.findNearestCommonDominator(From, To); - const TreeNodePtr NCDTN = DT.getNode(NCD); - const TreeNodePtr ToIDom = ToTN->getIDom(); - if (NCDTN != ToTN && NCDTN != ToIDom) { - errs() << "NearestCommonDominator verification failed:\n\tNCD(From:" - << BlockNamePrinter(From) << ", To:" << BlockNamePrinter(To) - << ") = " << BlockNamePrinter(NCD) - << ",\t (should be To or IDom[To]: " << BlockNamePrinter(ToIDom) - << ")\n"; + // Verify the root's DFS In number. Although DFS numbering would also work + // if we started from some other value, we assume 0-based numbering. + if (Root->getDFSNumIn() != 0) { + errs() << "DFSIn number for the tree root is not:\n\t"; + PrintNodeAndDFSNums(Root); + errs() << '\n'; + errs().flush(); + return false; + } + + // For each tree node verify if children's DFS numbers cover their parent's + // DFS numbers with no gaps. + for (const auto &NodeToTN : DT.DomTreeNodes) { + const TreeNodePtr Node = NodeToTN.second.get(); + + // Handle tree leaves. + if (Node->getChildren().empty()) { + if (Node->getDFSNumIn() + 1 != Node->getDFSNumOut()) { + errs() << "Tree leaf should have DFSOut = DFSIn + 1:\n\t"; + PrintNodeAndDFSNums(Node); + errs() << '\n'; + errs().flush(); + return false; + } + + continue; + } + + // Make a copy and sort it such that it is possible to check if there are + // no gaps between DFS numbers of adjacent children. + SmallVector<TreeNodePtr, 8> Children(Node->begin(), Node->end()); + std::sort(Children.begin(), Children.end(), + [](const TreeNodePtr Ch1, const TreeNodePtr Ch2) { + return Ch1->getDFSNumIn() < Ch2->getDFSNumIn(); + }); + + auto PrintChildrenError = [Node, &Children, PrintNodeAndDFSNums]( + const TreeNodePtr FirstCh, const TreeNodePtr SecondCh) { + assert(FirstCh); + + errs() << "Incorrect DFS numbers for:\n\tParent "; + PrintNodeAndDFSNums(Node); + + errs() << "\n\tChild "; + PrintNodeAndDFSNums(FirstCh); + + if (SecondCh) { + errs() << "\n\tSecond child "; + PrintNodeAndDFSNums(SecondCh); + } + + errs() << "\nAll children: "; + for (const TreeNodePtr Ch : Children) { + PrintNodeAndDFSNums(Ch); + errs() << ", "; + } + + errs() << '\n'; errs().flush(); + }; + if (Children.front()->getDFSNumIn() != Node->getDFSNumIn() + 1) { + PrintChildrenError(Children.front(), nullptr); return false; } + + if (Children.back()->getDFSNumOut() + 1 != Node->getDFSNumOut()) { + PrintChildrenError(Children.back(), nullptr); + return false; + } + + for (size_t i = 0, e = Children.size() - 1; i != e; ++i) { + if (Children[i]->getDFSNumOut() + 1 != Children[i + 1]->getDFSNumIn()) { + PrintChildrenError(Children[i], Children[i + 1]); + return false; + } + } } return true; @@ -888,6 +1510,8 @@ struct SemiNCAInfo { const NodePtr BB = TN->getBlock(); if (!BB || TN->getChildren().empty()) continue; + DEBUG(dbgs() << "Verifying parent property of node " + << BlockNamePrinter(TN) << "\n"); clear(); doFullDFSWalk(DT, [BB](NodePtr From, NodePtr To) { return From != BB && To != BB; @@ -945,33 +1569,37 @@ struct SemiNCAInfo { } }; - -template <class DomTreeT, class FuncT> -void Calculate(DomTreeT &DT, FuncT &F) { - SemiNCAInfo<DomTreeT> SNCA; - SNCA.calculateFromScratch(DT, GraphTraits<FuncT *>::size(&F)); +template <class DomTreeT> +void Calculate(DomTreeT &DT) { + SemiNCAInfo<DomTreeT>::CalculateFromScratch(DT, nullptr); } template <class DomTreeT> void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, typename DomTreeT::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo<DomTreeT>::InsertEdge(DT, From, To); + SemiNCAInfo<DomTreeT>::InsertEdge(DT, nullptr, From, To); } template <class DomTreeT> void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, typename DomTreeT::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo<DomTreeT>::DeleteEdge(DT, From, To); + SemiNCAInfo<DomTreeT>::DeleteEdge(DT, nullptr, From, To); +} + +template <class DomTreeT> +void ApplyUpdates(DomTreeT &DT, + ArrayRef<typename DomTreeT::UpdateType> Updates) { + SemiNCAInfo<DomTreeT>::ApplyUpdates(DT, Updates); } template <class DomTreeT> bool Verify(const DomTreeT &DT) { - SemiNCAInfo<DomTreeT> SNCA; - return SNCA.verifyReachability(DT) && SNCA.VerifyLevels(DT) && - SNCA.verifyNCD(DT) && SNCA.verifyParentProperty(DT) && - SNCA.verifySiblingProperty(DT); + SemiNCAInfo<DomTreeT> SNCA(nullptr); + return SNCA.verifyRoots(DT) && SNCA.verifyReachability(DT) && + SNCA.VerifyLevels(DT) && SNCA.verifyParentProperty(DT) && + SNCA.verifySiblingProperty(DT) && SNCA.VerifyDFSNumbers(DT); } } // namespace DomTreeBuilder diff --git a/contrib/llvm/include/llvm/Support/Host.h b/contrib/llvm/include/llvm/Support/Host.h index be93dd99032e..a4b0a340c568 100644 --- a/contrib/llvm/include/llvm/Support/Host.h +++ b/contrib/llvm/include/llvm/Support/Host.h @@ -15,7 +15,6 @@ #define LLVM_SUPPORT_HOST_H #include "llvm/ADT/StringMap.h" -#include "llvm/Support/MemoryBuffer.h" #if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) #include <endian.h> @@ -92,6 +91,7 @@ constexpr bool IsBigEndianHost = false; StringRef getHostCPUNameForPowerPC(const StringRef &ProcCpuinfoContent); StringRef getHostCPUNameForARM(const StringRef &ProcCpuinfoContent); StringRef getHostCPUNameForS390x(const StringRef &ProcCpuinfoContent); + StringRef getHostCPUNameForBPF(); } } } diff --git a/contrib/llvm/include/llvm/Support/KnownBits.h b/contrib/llvm/include/llvm/Support/KnownBits.h index 2c77d40559b9..7a4de3e5ff12 100644 --- a/contrib/llvm/include/llvm/Support/KnownBits.h +++ b/contrib/llvm/include/llvm/Support/KnownBits.h @@ -25,7 +25,7 @@ struct KnownBits { APInt One; private: - // Internal constructor for creating a ConstantRange from two APInts. + // Internal constructor for creating a KnownBits from two APInts. KnownBits(APInt Zero, APInt One) : Zero(std::move(Zero)), One(std::move(One)) {} @@ -193,6 +193,10 @@ public: unsigned countMaxPopulation() const { return getBitWidth() - Zero.countPopulation(); } + + /// Compute known bits resulting from adding LHS and RHS. + static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS, + KnownBits RHS); }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Support/LEB128.h b/contrib/llvm/include/llvm/Support/LEB128.h index 29640db69218..6af6e9f34474 100644 --- a/contrib/llvm/include/llvm/Support/LEB128.h +++ b/contrib/llvm/include/llvm/Support/LEB128.h @@ -21,23 +21,25 @@ namespace llvm { /// Utility function to encode a SLEB128 value to an output stream. inline void encodeSLEB128(int64_t Value, raw_ostream &OS, - unsigned Padding = 0) { + unsigned PadTo = 0) { bool More; + unsigned Count = 0; do { uint8_t Byte = Value & 0x7f; // NOTE: this assumes that this signed shift is an arithmetic right shift. Value >>= 7; More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || ((Value == -1) && ((Byte & 0x40) != 0)))); - if (More || Padding != 0) + Count++; + if (More || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. OS << char(Byte); } while (More); // Pad with 0x80 and emit a terminating byte at the end. - if (Padding != 0) { + if (Count < PadTo) { uint8_t PadValue = Value < 0 ? 0x7f : 0x00; - for (; Padding != 1; --Padding) + for (; Count < PadTo - 1; ++Count) OS << char(PadValue | 0x80); OS << char(PadValue); } @@ -45,8 +47,9 @@ inline void encodeSLEB128(int64_t Value, raw_ostream &OS, /// Utility function to encode a SLEB128 value to a buffer. Returns /// the length in bytes of the encoded value. -inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned Padding = 0) { +inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) { uint8_t *orig_p = p; + unsigned Count = 0; bool More; do { uint8_t Byte = Value & 0x7f; @@ -54,15 +57,16 @@ inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned Padding = 0) { Value >>= 7; More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || ((Value == -1) && ((Byte & 0x40) != 0)))); - if (More || Padding != 0) + Count++; + if (More || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. *p++ = Byte; } while (More); // Pad with 0x80 and emit a terminating byte at the end. - if (Padding != 0) { + if (Count < PadTo) { uint8_t PadValue = Value < 0 ? 0x7f : 0x00; - for (; Padding != 1; --Padding) + for (; Count < PadTo - 1; ++Count) *p++ = (PadValue | 0x80); *p++ = PadValue; } @@ -71,42 +75,48 @@ inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned Padding = 0) { /// Utility function to encode a ULEB128 value to an output stream. inline void encodeULEB128(uint64_t Value, raw_ostream &OS, - unsigned Padding = 0) { + unsigned PadTo = 0) { + unsigned Count = 0; do { uint8_t Byte = Value & 0x7f; Value >>= 7; - if (Value != 0 || Padding != 0) + Count++; + if (Value != 0 || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. OS << char(Byte); } while (Value != 0); // Pad with 0x80 and emit a null byte at the end. - if (Padding != 0) { - for (; Padding != 1; --Padding) + if (Count < PadTo) { + for (; Count < PadTo - 1; ++Count) OS << '\x80'; OS << '\x00'; + Count++; } } /// Utility function to encode a ULEB128 value to a buffer. Returns /// the length in bytes of the encoded value. inline unsigned encodeULEB128(uint64_t Value, uint8_t *p, - unsigned Padding = 0) { + unsigned PadTo = 0) { uint8_t *orig_p = p; + unsigned Count = 0; do { uint8_t Byte = Value & 0x7f; Value >>= 7; - if (Value != 0 || Padding != 0) + Count++; + if (Value != 0 || Count < PadTo) Byte |= 0x80; // Mark this byte to show that more bytes will follow. *p++ = Byte; } while (Value != 0); // Pad with 0x80 and emit a null byte at the end. - if (Padding != 0) { - for (; Padding != 1; --Padding) + if (Count < PadTo) { + for (; Count < PadTo - 1; ++Count) *p++ = '\x80'; *p++ = '\x00'; } + return (unsigned)(p - orig_p); } diff --git a/contrib/llvm/include/llvm/Support/LockFileManager.h b/contrib/llvm/include/llvm/Support/LockFileManager.h index 13d252425b93..1e417bdd5b25 100644 --- a/contrib/llvm/include/llvm/Support/LockFileManager.h +++ b/contrib/llvm/include/llvm/Support/LockFileManager.h @@ -11,6 +11,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/FileSystem.h" #include <system_error> #include <utility> // for std::pair @@ -53,10 +54,10 @@ public: private: SmallString<128> FileName; SmallString<128> LockFileName; - SmallString<128> UniqueLockFileName; + Optional<sys::fs::TempFile> UniqueLockFile; Optional<std::pair<std::string, int> > Owner; - Optional<std::error_code> Error; + std::error_code ErrorCode; std::string ErrorDiagMsg; LockFileManager(const LockFileManager &) = delete; @@ -88,8 +89,8 @@ public: std::string getErrorMessage() const; /// \brief Set error and error message - void setError(std::error_code &EC, StringRef ErrorMsg = "") { - Error = EC; + void setError(const std::error_code &EC, StringRef ErrorMsg = "") { + ErrorCode = EC; ErrorDiagMsg = ErrorMsg.str(); } }; diff --git a/contrib/llvm/include/llvm/Support/LowLevelTypeImpl.h b/contrib/llvm/include/llvm/Support/LowLevelTypeImpl.h index c79dd0c29507..099fa4618997 100644 --- a/contrib/llvm/include/llvm/Support/LowLevelTypeImpl.h +++ b/contrib/llvm/include/llvm/Support/LowLevelTypeImpl.h @@ -137,51 +137,6 @@ public: return scalar(getScalarSizeInBits()); } - /// Get a low-level type with half the size of the original, by halving the - /// size of the scalar type involved. For example `s32` will become `s16`, - /// `<2 x s32>` will become `<2 x s16>`. - LLT halfScalarSize() const { - assert(!IsPointer && getScalarSizeInBits() > 1 && - getScalarSizeInBits() % 2 == 0 && "cannot half size of this type"); - return LLT{/*isPointer=*/false, IsVector ? true : false, - IsVector ? getNumElements() : (uint16_t)0, - getScalarSizeInBits() / 2, /*AddressSpace=*/0}; - } - - /// Get a low-level type with twice the size of the original, by doubling the - /// size of the scalar type involved. For example `s32` will become `s64`, - /// `<2 x s32>` will become `<2 x s64>`. - LLT doubleScalarSize() const { - assert(!IsPointer && "cannot change size of this type"); - return LLT{/*isPointer=*/false, IsVector ? true : false, - IsVector ? getNumElements() : (uint16_t)0, - getScalarSizeInBits() * 2, /*AddressSpace=*/0}; - } - - /// Get a low-level type with half the size of the original, by halving the - /// number of vector elements of the scalar type involved. The source must be - /// a vector type with an even number of elements. For example `<4 x s32>` - /// will become `<2 x s32>`, `<2 x s32>` will become `s32`. - LLT halfElements() const { - assert(isVector() && getNumElements() % 2 == 0 && "cannot half odd vector"); - if (getNumElements() == 2) - return scalar(getScalarSizeInBits()); - - return LLT{/*isPointer=*/false, /*isVector=*/true, - (uint16_t)(getNumElements() / 2), getScalarSizeInBits(), - /*AddressSpace=*/0}; - } - - /// Get a low-level type with twice the size of the original, by doubling the - /// number of vector elements of the scalar type involved. The source must be - /// a vector type. For example `<2 x s32>` will become `<4 x s32>`. Doubling - /// the number of elements in sN produces <2 x sN>. - LLT doubleElements() const { - return LLT{IsPointer ? true : false, /*isVector=*/true, - (uint16_t)(getNumElements() * 2), getScalarSizeInBits(), - IsPointer ? getAddressSpace() : 0}; - } - void print(raw_ostream &OS) const; bool operator==(const LLT &RHS) const { diff --git a/contrib/llvm/include/llvm/Support/MathExtras.h b/contrib/llvm/include/llvm/Support/MathExtras.h index fd29865c8475..a37a16784e2a 100644 --- a/contrib/llvm/include/llvm/Support/MathExtras.h +++ b/contrib/llvm/include/llvm/Support/MathExtras.h @@ -424,7 +424,7 @@ constexpr inline bool isPowerOf2_32(uint32_t Value) { /// Return true if the argument is a power of two > 0 (64 bit edition.) constexpr inline bool isPowerOf2_64(uint64_t Value) { - return Value && !(Value & (Value - int64_t(1L))); + return Value && !(Value & (Value - 1)); } /// Return a byte-swapped representation of the 16-bit argument. @@ -687,6 +687,11 @@ template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) { return (Value + Align - 1) / Align * Align; } +/// Returns the integer ceil(Numerator / Denominator). +inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) { + return alignTo(Numerator, Denominator) / Denominator; +} + /// \c alignTo for contexts where a constant expression is required. /// \sa alignTo /// diff --git a/contrib/llvm/include/llvm/Support/Memory.h b/contrib/llvm/include/llvm/Support/Memory.h index 8103aea2fa25..3140dc6eef42 100644 --- a/contrib/llvm/include/llvm/Support/Memory.h +++ b/contrib/llvm/include/llvm/Support/Memory.h @@ -109,51 +109,10 @@ namespace sys { static std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags); - /// This method allocates a block of Read/Write/Execute memory that is - /// suitable for executing dynamically generated code (e.g. JIT). An - /// attempt to allocate \p NumBytes bytes of virtual memory is made. - /// \p NearBlock may point to an existing allocation in which case - /// an attempt is made to allocate more memory near the existing block. - /// - /// On success, this returns a non-null memory block, otherwise it returns - /// a null memory block and fills in *ErrMsg. - /// - /// @brief Allocate Read/Write/Execute memory. - static MemoryBlock AllocateRWX(size_t NumBytes, - const MemoryBlock *NearBlock, - std::string *ErrMsg = nullptr); - - /// This method releases a block of Read/Write/Execute memory that was - /// allocated with the AllocateRWX method. It should not be used to - /// release any memory block allocated any other way. - /// - /// On success, this returns false, otherwise it returns true and fills - /// in *ErrMsg. - /// @brief Release Read/Write/Execute memory. - static bool ReleaseRWX(MemoryBlock &block, std::string *ErrMsg = nullptr); - /// InvalidateInstructionCache - Before the JIT can run a block of code /// that has been emitted it must invalidate the instruction cache on some /// platforms. static void InvalidateInstructionCache(const void *Addr, size_t Len); - - /// setExecutable - Before the JIT can run a block of code, it has to be - /// given read and executable privilege. Return true if it is already r-x - /// or the system is able to change its previlege. - static bool setExecutable(MemoryBlock &M, std::string *ErrMsg = nullptr); - - /// setWritable - When adding to a block of code, the JIT may need - /// to mark a block of code as RW since the protections are on page - /// boundaries, and the JIT internal allocations are not page aligned. - static bool setWritable(MemoryBlock &M, std::string *ErrMsg = nullptr); - - /// setRangeExecutable - Mark the page containing a range of addresses - /// as executable. - static bool setRangeExecutable(const void *Addr, size_t Size); - - /// setRangeWritable - Mark the page containing a range of addresses - /// as writable. - static bool setRangeWritable(const void *Addr, size_t Size); }; /// Owning version of MemoryBlock. diff --git a/contrib/llvm/include/llvm/Support/MemoryBuffer.h b/contrib/llvm/include/llvm/Support/MemoryBuffer.h index 73f0251a6b6e..59c93f15d7b8 100644 --- a/contrib/llvm/include/llvm/Support/MemoryBuffer.h +++ b/contrib/llvm/include/llvm/Support/MemoryBuffer.h @@ -136,7 +136,8 @@ public: /// Map a subrange of the specified file as a MemoryBuffer. static ErrorOr<std::unique_ptr<MemoryBuffer>> - getFileSlice(const Twine &Filename, uint64_t MapSize, uint64_t Offset, bool IsVolatile = false); + getFileSlice(const Twine &Filename, uint64_t MapSize, uint64_t Offset, + bool IsVolatile = false); //===--------------------------------------------------------------------===// // Provided for performance analysis. diff --git a/contrib/llvm/include/llvm/Support/Parallel.h b/contrib/llvm/include/llvm/Support/Parallel.h index e36e0cc29e14..6bc0a6bbaf2b 100644 --- a/contrib/llvm/include/llvm/Support/Parallel.h +++ b/contrib/llvm/include/llvm/Support/Parallel.h @@ -158,11 +158,11 @@ void parallel_for_each(IterTy Begin, IterTy End, FuncTy Fn) { TaskSize = 1; TaskGroup TG; - while (TaskSize <= std::distance(Begin, End)) { + while (TaskSize < std::distance(Begin, End)) { TG.spawn([=, &Fn] { std::for_each(Begin, Begin + TaskSize, Fn); }); Begin += TaskSize; } - TG.spawn([=, &Fn] { std::for_each(Begin, End, Fn); }); + std::for_each(Begin, End, Fn); } template <class IndexTy, class FuncTy> @@ -179,10 +179,8 @@ void parallel_for_each_n(IndexTy Begin, IndexTy End, FuncTy Fn) { Fn(J); }); } - TG.spawn([=, &Fn] { - for (IndexTy J = I; J < End; ++J) - Fn(J); - }); + for (IndexTy J = I; J < End; ++J) + Fn(J); } #endif diff --git a/contrib/llvm/include/llvm/Support/PointerLikeTypeTraits.h b/contrib/llvm/include/llvm/Support/PointerLikeTypeTraits.h index 521a49684e45..794230d606a4 100644 --- a/contrib/llvm/include/llvm/Support/PointerLikeTypeTraits.h +++ b/contrib/llvm/include/llvm/Support/PointerLikeTypeTraits.h @@ -22,11 +22,7 @@ namespace llvm { /// A traits type that is used to handle pointer types and things that are just /// wrappers for pointers as a uniform entity. -template <typename T> class PointerLikeTypeTraits { - // getAsVoidPointer - // getFromVoidPointer - // getNumLowBitsAvailable -}; +template <typename T> struct PointerLikeTypeTraits; namespace detail { /// A tiny meta function to compute the log2 of a compile time constant. @@ -34,19 +30,36 @@ template <size_t N> struct ConstantLog2 : std::integral_constant<size_t, ConstantLog2<N / 2>::value + 1> {}; template <> struct ConstantLog2<1> : std::integral_constant<size_t, 0> {}; -} + +// Provide a trait to check if T is pointer-like. +template <typename T, typename U = void> struct HasPointerLikeTypeTraits { + static const bool value = false; +}; + +// sizeof(T) is valid only for a complete T. +template <typename T> struct HasPointerLikeTypeTraits< + T, decltype((sizeof(PointerLikeTypeTraits<T>) + sizeof(T)), void())> { + static const bool value = true; +}; + +template <typename T> struct IsPointerLike { + static const bool value = HasPointerLikeTypeTraits<T>::value; +}; + +template <typename T> struct IsPointerLike<T *> { + static const bool value = true; +}; +} // namespace detail // Provide PointerLikeTypeTraits for non-cvr pointers. -template <typename T> class PointerLikeTypeTraits<T *> { -public: +template <typename T> struct PointerLikeTypeTraits<T *> { static inline void *getAsVoidPointer(T *P) { return P; } static inline T *getFromVoidPointer(void *P) { return static_cast<T *>(P); } enum { NumLowBitsAvailable = detail::ConstantLog2<alignof(T)>::value }; }; -template <> class PointerLikeTypeTraits<void *> { -public: +template <> struct PointerLikeTypeTraits<void *> { static inline void *getAsVoidPointer(void *P) { return P; } static inline void *getFromVoidPointer(void *P) { return P; } @@ -61,10 +74,9 @@ public: }; // Provide PointerLikeTypeTraits for const things. -template <typename T> class PointerLikeTypeTraits<const T> { +template <typename T> struct PointerLikeTypeTraits<const T> { typedef PointerLikeTypeTraits<T> NonConst; -public: static inline const void *getAsVoidPointer(const T P) { return NonConst::getAsVoidPointer(P); } @@ -75,10 +87,9 @@ public: }; // Provide PointerLikeTypeTraits for const pointers. -template <typename T> class PointerLikeTypeTraits<const T *> { +template <typename T> struct PointerLikeTypeTraits<const T *> { typedef PointerLikeTypeTraits<T *> NonConst; -public: static inline const void *getAsVoidPointer(const T *P) { return NonConst::getAsVoidPointer(const_cast<T *>(P)); } @@ -89,8 +100,7 @@ public: }; // Provide PointerLikeTypeTraits for uintptr_t. -template <> class PointerLikeTypeTraits<uintptr_t> { -public: +template <> struct PointerLikeTypeTraits<uintptr_t> { static inline void *getAsVoidPointer(uintptr_t P) { return reinterpret_cast<void *>(P); } diff --git a/contrib/llvm/include/llvm/Support/Printable.h b/contrib/llvm/include/llvm/Support/Printable.h index 28e875e8ff5e..cb55d41316e3 100644 --- a/contrib/llvm/include/llvm/Support/Printable.h +++ b/contrib/llvm/include/llvm/Support/Printable.h @@ -42,7 +42,7 @@ public: : Print(std::move(Print)) {} }; -static inline raw_ostream &operator<<(raw_ostream &OS, const Printable &P) { +inline raw_ostream &operator<<(raw_ostream &OS, const Printable &P) { P.Print(OS); return OS; } diff --git a/contrib/llvm/include/llvm/Support/Process.h b/contrib/llvm/include/llvm/Support/Process.h index 780c7e2ddd6f..82b0d9f6ba28 100644 --- a/contrib/llvm/include/llvm/Support/Process.h +++ b/contrib/llvm/include/llvm/Support/Process.h @@ -80,9 +80,15 @@ public: /// This function searches for an existing file in the list of directories /// in a PATH like environment variable, and returns the first file found, /// according to the order of the entries in the PATH like environment - /// variable. - static Optional<std::string> FindInEnvPath(const std::string& EnvName, - const std::string& FileName); + /// variable. If an ignore list is specified, then any folder which is in + /// the PATH like environment variable but is also in IgnoreList is not + /// considered. + static Optional<std::string> FindInEnvPath(StringRef EnvName, + StringRef FileName, + ArrayRef<std::string> IgnoreList); + + static Optional<std::string> FindInEnvPath(StringRef EnvName, + StringRef FileName); /// This function returns a SmallVector containing the arguments passed from /// the operating system to the program. This function expects to be handed diff --git a/contrib/llvm/include/llvm/Support/Program.h b/contrib/llvm/include/llvm/Support/Program.h index 055f016d8243..06fd35078145 100644 --- a/contrib/llvm/include/llvm/Support/Program.h +++ b/contrib/llvm/include/llvm/Support/Program.h @@ -15,12 +15,12 @@ #define LLVM_SUPPORT_PROGRAM_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/ErrorOr.h" #include <system_error> namespace llvm { -class StringRef; - namespace sys { /// This is the OS-specific separator for PATH like environment variables: @@ -69,7 +69,7 @@ struct ProcessInfo { /// \returns The fully qualified path to the first \p Name in \p Paths if it /// exists. \p Name if \p Name has slashes in it. Otherwise an error. ErrorOr<std::string> - findProgramByName(StringRef Name, ArrayRef<StringRef> Paths = None); + findProgramByName(StringRef Name, ArrayRef<StringRef> Paths = {}); // These functions change the specified standard stream (stdin or stdout) to // binary mode. They return errc::success if the specified stream @@ -84,33 +84,33 @@ struct ProcessInfo { /// This function waits for the program to finish, so should be avoided in /// library functions that aren't expected to block. Consider using /// ExecuteNoWait() instead. - /// @returns an integer result code indicating the status of the program. + /// \returns an integer result code indicating the status of the program. /// A zero or positive value indicates the result code of the program. /// -1 indicates failure to execute /// -2 indicates a crash during execution or timeout int ExecuteAndWait( StringRef Program, ///< Path of the program to be executed. It is ///< presumed this is the result of the findProgramByName method. - const char **args, ///< A vector of strings that are passed to the + const char **Args, ///< A vector of strings that are passed to the ///< program. The first element should be the name of the program. ///< The list *must* be terminated by a null char* entry. - const char **env = nullptr, ///< An optional vector of strings to use for + const char **Env = nullptr, ///< An optional vector of strings to use for ///< the program's environment. If not provided, the current program's ///< environment will be used. - const StringRef **redirects = nullptr, ///< An optional array of pointers - ///< to paths. If the array is null, no redirection is done. The array - ///< should have a size of at least three. The inferior process's - ///< stdin(0), stdout(1), and stderr(2) will be redirected to the - ///< corresponding paths. - ///< When an empty path is passed in, the corresponding file - ///< descriptor will be disconnected (ie, /dev/null'd) in a portable - ///< way. - unsigned secondsToWait = 0, ///< If non-zero, this specifies the amount + ArrayRef<Optional<StringRef>> Redirects = {}, ///< + ///< An array of optional paths. Should have a size of zero or three. + ///< If the array is empty, no redirections are performed. + ///< Otherwise, the inferior process's stdin(0), stdout(1), and stderr(2) + ///< will be redirected to the corresponding paths, if the optional path + ///< is present (not \c llvm::None). + ///< When an empty path is passed in, the corresponding file descriptor + ///< will be disconnected (ie, /dev/null'd) in a portable way. + unsigned SecondsToWait = 0, ///< If non-zero, this specifies the amount ///< of time to wait for the child process to exit. If the time ///< expires, the child is killed and this call returns. If zero, ///< this function will wait until the child finishes or forever if ///< it doesn't. - unsigned memoryLimit = 0, ///< If non-zero, this specifies max. amount + unsigned MemoryLimit = 0, ///< If non-zero, this specifies max. amount ///< of memory can be allocated by process. If memory usage will be ///< higher limit, the child is killed and this call returns. If zero ///< - no memory limit. @@ -122,17 +122,20 @@ struct ProcessInfo { /// Similar to ExecuteAndWait, but returns immediately. /// @returns The \see ProcessInfo of the newly launced process. - /// \note On Microsoft Windows systems, users will need to either call \see - /// Wait until the process finished execution or win32 CloseHandle() API on - /// ProcessInfo.ProcessHandle to avoid memory leaks. - ProcessInfo - ExecuteNoWait(StringRef Program, const char **args, const char **env = nullptr, - const StringRef **redirects = nullptr, unsigned memoryLimit = 0, - std::string *ErrMsg = nullptr, bool *ExecutionFailed = nullptr); + /// \note On Microsoft Windows systems, users will need to either call + /// \see Wait until the process finished execution or win32 CloseHandle() API + /// on ProcessInfo.ProcessHandle to avoid memory leaks. + ProcessInfo ExecuteNoWait(StringRef Program, const char **Args, + const char **Env = nullptr, + ArrayRef<Optional<StringRef>> Redirects = {}, + unsigned MemoryLimit = 0, + std::string *ErrMsg = nullptr, + bool *ExecutionFailed = nullptr); /// Return true if the given arguments fit within system-specific /// argument length limits. - bool commandLineFitsWithinSystemLimits(StringRef Program, ArrayRef<const char*> Args); + bool commandLineFitsWithinSystemLimits(StringRef Program, + ArrayRef<const char *> Args); /// File encoding options when writing contents that a non-UTF8 tool will /// read (on Windows systems). For UNIX, we always use UTF-8. diff --git a/contrib/llvm/include/llvm/Support/ReverseIteration.h b/contrib/llvm/include/llvm/Support/ReverseIteration.h index cb97b60f06dd..5e0238d81c4c 100644 --- a/contrib/llvm/include/llvm/Support/ReverseIteration.h +++ b/contrib/llvm/include/llvm/Support/ReverseIteration.h @@ -2,16 +2,18 @@ #define LLVM_SUPPORT_REVERSEITERATION_H #include "llvm/Config/abi-breaking.h" +#include "llvm/Support/PointerLikeTypeTraits.h" namespace llvm { -#if LLVM_ENABLE_ABI_BREAKING_CHECKS -template <class T = void> struct ReverseIterate { static bool value; }; + +template<class T = void *> +bool shouldReverseIterate() { #if LLVM_ENABLE_REVERSE_ITERATION -template <class T> bool ReverseIterate<T>::value = true; + return detail::IsPointerLike<T>::value; #else -template <class T> bool ReverseIterate<T>::value = false; -#endif + return false; #endif } +} #endif diff --git a/contrib/llvm/include/llvm/Support/ScaledNumber.h b/contrib/llvm/include/llvm/Support/ScaledNumber.h index 910174732994..cfbdbc751617 100644 --- a/contrib/llvm/include/llvm/Support/ScaledNumber.h +++ b/contrib/llvm/include/llvm/Support/ScaledNumber.h @@ -504,13 +504,13 @@ private: static_assert(Width <= 64, "invalid integer width for digits"); private: - DigitsType Digits; - int16_t Scale; + DigitsType Digits = 0; + int16_t Scale = 0; public: - ScaledNumber() : Digits(0), Scale(0) {} + ScaledNumber() = default; - ScaledNumber(DigitsType Digits, int16_t Scale) + constexpr ScaledNumber(DigitsType Digits, int16_t Scale) : Digits(Digits), Scale(Scale) {} private: diff --git a/contrib/llvm/include/llvm/Support/SourceMgr.h b/contrib/llvm/include/llvm/Support/SourceMgr.h index 399f8dcd76fc..c08bf858760a 100644 --- a/contrib/llvm/include/llvm/Support/SourceMgr.h +++ b/contrib/llvm/include/llvm/Support/SourceMgr.h @@ -43,7 +43,8 @@ public: enum DiagKind { DK_Error, DK_Warning, - DK_Note + DK_Remark, + DK_Note, }; /// Clients that want to handle their own diagnostics in a custom way can diff --git a/contrib/llvm/include/llvm/Support/SpecialCaseList.h b/contrib/llvm/include/llvm/Support/SpecialCaseList.h index ce693c501312..fd62fc48047b 100644 --- a/contrib/llvm/include/llvm/Support/SpecialCaseList.h +++ b/contrib/llvm/include/llvm/Support/SpecialCaseList.h @@ -8,15 +8,19 @@ // // This is a utility class used to parse user-provided text files with // "special case lists" for code sanitizers. Such files are used to -// define "ABI list" for DataFlowSanitizer and blacklists for another sanitizers +// define an "ABI list" for DataFlowSanitizer and blacklists for sanitizers // like AddressSanitizer or UndefinedBehaviorSanitizer. // -// Empty lines and lines starting with "#" are ignored. All the rest lines -// should have the form: -// section:wildcard_expression[=category] +// Empty lines and lines starting with "#" are ignored. Sections are defined +// using a '[section_name]' header and can be used to specify sanitizers the +// entries below it apply to. Section names are regular expressions, and +// entries without a section header match all sections (e.g. an '[*]' header +// is assumed.) +// The remaining lines should have the form: +// prefix:wildcard_expression[=category] // If category is not specified, it is assumed to be empty string. -// Definitions of "section" and "category" are sanitizer-specific. For example, -// sanitizer blacklists support sections "src", "fun" and "global". +// Definitions of "prefix" and "category" are sanitizer-specific. For example, +// sanitizer blacklists support prefixes "src", "fun" and "global". // Wildcard expressions define, respectively, source files, functions or // globals which shouldn't be instrumented. // Examples of categories: @@ -26,6 +30,7 @@ // detection for certain globals or source files. // Full special case list file example: // --- +// [address] // # Blacklisted items: // fun:*_ZN4base6subtle* // global:*global_with_bad_access_or_initialization* @@ -34,14 +39,13 @@ // src:file_with_tricky_code.cc // src:ignore-global-initializers-issues.cc=init // +// [dataflow] // # Functions with pure functional semantics: // fun:cos=functional // fun:sin=functional // --- // Note that the wild card is in fact an llvm::Regex, but * is automatically // replaced with .* -// This is similar to the "ignore" feature of ThreadSanitizer. -// http://code.google.com/p/data-race-test/wiki/ThreadSanitizerIgnores // //===----------------------------------------------------------------------===// @@ -49,6 +53,9 @@ #define LLVM_SUPPORT_SPECIALCASELIST_H #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/TrigramIndex.h" #include <string> #include <vector> @@ -76,26 +83,70 @@ public: /// Returns true, if special case list contains a line /// \code - /// @Section:<E>=@Category + /// @Prefix:<E>=@Category /// \endcode - /// and @Query satisfies a wildcard expression <E>. - bool inSection(StringRef Section, StringRef Query, + /// where @Query satisfies wildcard expression <E> in a given @Section. + bool inSection(StringRef Section, StringRef Prefix, StringRef Query, StringRef Category = StringRef()) const; -private: + /// Returns the line number corresponding to the special case list entry if + /// the special case list contains a line + /// \code + /// @Prefix:<E>=@Category + /// \endcode + /// where @Query satisfies wildcard expression <E> in a given @Section. + /// Returns zero if there is no blacklist entry corresponding to this + /// expression. + unsigned inSectionBlame(StringRef Section, StringRef Prefix, StringRef Query, + StringRef Category = StringRef()) const; + +protected: + // Implementations of the create*() functions that can also be used by derived + // classes. + bool createInternal(const std::vector<std::string> &Paths, + std::string &Error); + bool createInternal(const MemoryBuffer *MB, std::string &Error); + + SpecialCaseList() = default; SpecialCaseList(SpecialCaseList const &) = delete; SpecialCaseList &operator=(SpecialCaseList const &) = delete; - struct Entry; - StringMap<StringMap<Entry>> Entries; - StringMap<StringMap<std::string>> Regexps; - bool IsCompiled; + /// Represents a set of regular expressions. Regular expressions which are + /// "literal" (i.e. no regex metacharacters) are stored in Strings. The + /// reason for doing so is efficiency; StringMap is much faster at matching + /// literal strings than Regex. + class Matcher { + public: + bool insert(std::string Regexp, unsigned LineNumber, std::string &REError); + // Returns the line number in the source file that this query matches to. + // Returns zero if no match is found. + unsigned match(StringRef Query) const; + + private: + StringMap<unsigned> Strings; + TrigramIndex Trigrams; + std::vector<std::pair<std::unique_ptr<Regex>, unsigned>> RegExes; + }; + + using SectionEntries = StringMap<StringMap<Matcher>>; + + struct Section { + Section(std::unique_ptr<Matcher> M) : SectionMatcher(std::move(M)){}; + + std::unique_ptr<Matcher> SectionMatcher; + SectionEntries Entries; + }; + + std::vector<Section> Sections; - SpecialCaseList(); /// Parses just-constructed SpecialCaseList entries from a memory buffer. - bool parse(const MemoryBuffer *MB, std::string &Error); - /// compile() should be called once, after parsing all the memory buffers. - void compile(); + bool parse(const MemoryBuffer *MB, StringMap<size_t> &SectionsMap, + std::string &Error); + + // Helper method for derived classes to search by Prefix, Query, and Category + // once they have already resolved a section entry. + unsigned inSectionBlame(const SectionEntries &Entries, StringRef Prefix, + StringRef Query, StringRef Category) const; }; } // namespace llvm diff --git a/contrib/llvm/include/llvm/Support/TarWriter.h b/contrib/llvm/include/llvm/Support/TarWriter.h index 44bdcaf2c465..639f61b53892 100644 --- a/contrib/llvm/include/llvm/Support/TarWriter.h +++ b/contrib/llvm/include/llvm/Support/TarWriter.h @@ -11,6 +11,7 @@ #define LLVM_SUPPORT_TAR_WRITER_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Support/Error.h" #include "llvm/Support/raw_ostream.h" @@ -26,6 +27,7 @@ private: TarWriter(int FD, StringRef BaseDir); raw_fd_ostream OS; std::string BaseDir; + StringSet<> Files; }; } diff --git a/contrib/llvm/include/llvm/Support/TargetParser.h b/contrib/llvm/include/llvm/Support/TargetParser.h index e13582f6a6d3..13b7befb8ce4 100644 --- a/contrib/llvm/include/llvm/Support/TargetParser.h +++ b/contrib/llvm/include/llvm/Support/TargetParser.h @@ -31,61 +31,61 @@ class StringRef; // back-end to TableGen to create these clean tables. namespace ARM { -// FPU names. -enum FPUKind { -#define ARM_FPU(NAME, KIND, VERSION, NEON_SUPPORT, RESTRICTION) KIND, -#include "ARMTargetParser.def" - FK_LAST +// FPU Version +enum class FPUVersion { + NONE, + VFPV2, + VFPV3, + VFPV3_FP16, + VFPV4, + VFPV5 }; -// FPU Version -enum FPUVersion { - FV_NONE = 0, - FV_VFPV2, - FV_VFPV3, - FV_VFPV3_FP16, - FV_VFPV4, - FV_VFPV5 +// An FPU name restricts the FPU in one of three ways: +enum class FPURestriction { + None = 0, ///< No restriction + D16, ///< Only 16 D registers + SP_D16 ///< Only single-precision instructions, with 16 D registers }; // An FPU name implies one of three levels of Neon support: -enum NeonSupportLevel { - NS_None = 0, ///< No Neon - NS_Neon, ///< Neon - NS_Crypto ///< Neon with Crypto +enum class NeonSupportLevel { + None = 0, ///< No Neon + Neon, ///< Neon + Crypto ///< Neon with Crypto }; -// An FPU name restricts the FPU in one of three ways: -enum FPURestriction { - FR_None = 0, ///< No restriction - FR_D16, ///< Only 16 D registers - FR_SP_D16 ///< Only single-precision instructions, with 16 D registers +// FPU names. +enum FPUKind { +#define ARM_FPU(NAME, KIND, VERSION, NEON_SUPPORT, RESTRICTION) KIND, +#include "ARMTargetParser.def" + FK_LAST }; // Arch names. -enum ArchKind { +enum class ArchKind { #define ARM_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) ID, #include "ARMTargetParser.def" - AK_LAST }; // Arch extension modifiers for CPUs. enum ArchExtKind : unsigned { - AEK_INVALID = 0x0, - AEK_NONE = 0x1, - AEK_CRC = 0x2, - AEK_CRYPTO = 0x4, - AEK_FP = 0x8, - AEK_HWDIVTHUMB = 0x10, - AEK_HWDIVARM = 0x20, - AEK_MP = 0x40, - AEK_SIMD = 0x80, - AEK_SEC = 0x100, - AEK_VIRT = 0x200, - AEK_DSP = 0x400, - AEK_FP16 = 0x800, - AEK_RAS = 0x1000, - AEK_SVE = 0x2000, + AEK_INVALID = 0, + AEK_NONE = 1, + AEK_CRC = 1 << 1, + AEK_CRYPTO = 1 << 2, + AEK_FP = 1 << 3, + AEK_HWDIVTHUMB = 1 << 4, + AEK_HWDIVARM = 1 << 5, + AEK_MP = 1 << 6, + AEK_SIMD = 1 << 7, + AEK_SEC = 1 << 8, + AEK_VIRT = 1 << 9, + AEK_DSP = 1 << 10, + AEK_FP16 = 1 << 11, + AEK_RAS = 1 << 12, + AEK_SVE = 1 << 13, + AEK_DOTPROD = 1 << 14, // Unsupported extensions. AEK_OS = 0x8000000, AEK_IWMMXT = 0x10000000, @@ -95,22 +95,22 @@ enum ArchExtKind : unsigned { }; // ISA kinds. -enum ISAKind { IK_INVALID = 0, IK_ARM, IK_THUMB, IK_AARCH64 }; +enum class ISAKind { INVALID = 0, ARM, THUMB, AARCH64 }; // Endianness // FIXME: BE8 vs. BE32? -enum EndianKind { EK_INVALID = 0, EK_LITTLE, EK_BIG }; +enum class EndianKind { INVALID = 0, LITTLE, BIG }; // v6/v7/v8 Profile -enum ProfileKind { PK_INVALID = 0, PK_A, PK_R, PK_M }; +enum class ProfileKind { INVALID = 0, A, R, M }; StringRef getCanonicalArchName(StringRef Arch); // Information by ID StringRef getFPUName(unsigned FPUKind); -unsigned getFPUVersion(unsigned FPUKind); -unsigned getFPUNeonSupportLevel(unsigned FPUKind); -unsigned getFPURestriction(unsigned FPUKind); +FPUVersion getFPUVersion(unsigned FPUKind); +NeonSupportLevel getFPUNeonSupportLevel(unsigned FPUKind); +FPURestriction getFPURestriction(unsigned FPUKind); // FIXME: These should be moved to TargetTuple once it exists bool getFPUFeatures(unsigned FPUKind, std::vector<StringRef> &Features); @@ -118,28 +118,28 @@ bool getHWDivFeatures(unsigned HWDivKind, std::vector<StringRef> &Features); bool getExtensionFeatures(unsigned Extensions, std::vector<StringRef> &Features); -StringRef getArchName(unsigned ArchKind); -unsigned getArchAttr(unsigned ArchKind); -StringRef getCPUAttr(unsigned ArchKind); -StringRef getSubArch(unsigned ArchKind); +StringRef getArchName(ArchKind AK); +unsigned getArchAttr(ArchKind AK); +StringRef getCPUAttr(ArchKind AK); +StringRef getSubArch(ArchKind AK); StringRef getArchExtName(unsigned ArchExtKind); StringRef getArchExtFeature(StringRef ArchExt); StringRef getHWDivName(unsigned HWDivKind); // Information by Name -unsigned getDefaultFPU(StringRef CPU, unsigned ArchKind); -unsigned getDefaultExtensions(StringRef CPU, unsigned ArchKind); +unsigned getDefaultFPU(StringRef CPU, ArchKind AK); +unsigned getDefaultExtensions(StringRef CPU, ArchKind AK); StringRef getDefaultCPU(StringRef Arch); // Parser unsigned parseHWDiv(StringRef HWDiv); unsigned parseFPU(StringRef FPU); -unsigned parseArch(StringRef Arch); +ArchKind parseArch(StringRef Arch); unsigned parseArchExt(StringRef ArchExt); -unsigned parseCPUArch(StringRef CPU); -unsigned parseArchISA(StringRef Arch); -unsigned parseArchEndian(StringRef Arch); -unsigned parseArchProfile(StringRef Arch); +ArchKind parseCPUArch(StringRef CPU); +ISAKind parseArchISA(StringRef Arch); +EndianKind parseArchEndian(StringRef Arch); +ProfileKind parseArchProfile(StringRef Arch); unsigned parseArchVersion(StringRef Arch); StringRef computeDefaultTargetABI(const Triple &TT, StringRef CPU); @@ -153,62 +153,108 @@ namespace AArch64 { enum class ArchKind { #define AARCH64_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) ID, #include "AArch64TargetParser.def" - AK_LAST }; // Arch extension modifiers for CPUs. enum ArchExtKind : unsigned { - AEK_INVALID = 0x0, - AEK_NONE = 0x1, - AEK_CRC = 0x2, - AEK_CRYPTO = 0x4, - AEK_FP = 0x8, - AEK_SIMD = 0x10, - AEK_FP16 = 0x20, - AEK_PROFILE = 0x40, - AEK_RAS = 0x80, - AEK_LSE = 0x100, - AEK_SVE = 0x200 + AEK_INVALID = 0, + AEK_NONE = 1, + AEK_CRC = 1 << 1, + AEK_CRYPTO = 1 << 2, + AEK_FP = 1 << 3, + AEK_SIMD = 1 << 4, + AEK_FP16 = 1 << 5, + AEK_PROFILE = 1 << 6, + AEK_RAS = 1 << 7, + AEK_LSE = 1 << 8, + AEK_SVE = 1 << 9, + AEK_DOTPROD = 1 << 10, + AEK_RCPC = 1 << 11, + AEK_RDM = 1 << 12 }; StringRef getCanonicalArchName(StringRef Arch); // Information by ID StringRef getFPUName(unsigned FPUKind); -unsigned getFPUVersion(unsigned FPUKind); -unsigned getFPUNeonSupportLevel(unsigned FPUKind); -unsigned getFPURestriction(unsigned FPUKind); +ARM::FPUVersion getFPUVersion(unsigned FPUKind); +ARM::NeonSupportLevel getFPUNeonSupportLevel(unsigned FPUKind); +ARM::FPURestriction getFPURestriction(unsigned FPUKind); // FIXME: These should be moved to TargetTuple once it exists bool getFPUFeatures(unsigned FPUKind, std::vector<StringRef> &Features); bool getExtensionFeatures(unsigned Extensions, std::vector<StringRef> &Features); -bool getArchFeatures(unsigned ArchKind, std::vector<StringRef> &Features); +bool getArchFeatures(ArchKind AK, std::vector<StringRef> &Features); -StringRef getArchName(unsigned ArchKind); -unsigned getArchAttr(unsigned ArchKind); -StringRef getCPUAttr(unsigned ArchKind); -StringRef getSubArch(unsigned ArchKind); +StringRef getArchName(ArchKind AK); +unsigned getArchAttr(ArchKind AK); +StringRef getCPUAttr(ArchKind AK); +StringRef getSubArch(ArchKind AK); StringRef getArchExtName(unsigned ArchExtKind); StringRef getArchExtFeature(StringRef ArchExt); unsigned checkArchVersion(StringRef Arch); // Information by Name -unsigned getDefaultFPU(StringRef CPU, unsigned ArchKind); -unsigned getDefaultExtensions(StringRef CPU, unsigned ArchKind); +unsigned getDefaultFPU(StringRef CPU, ArchKind AK); +unsigned getDefaultExtensions(StringRef CPU, ArchKind AK); StringRef getDefaultCPU(StringRef Arch); // Parser unsigned parseFPU(StringRef FPU); -unsigned parseArch(StringRef Arch); +AArch64::ArchKind parseArch(StringRef Arch); unsigned parseArchExt(StringRef ArchExt); -unsigned parseCPUArch(StringRef CPU); -unsigned parseArchISA(StringRef Arch); -unsigned parseArchEndian(StringRef Arch); -unsigned parseArchProfile(StringRef Arch); +ArchKind parseCPUArch(StringRef CPU); +ARM::ISAKind parseArchISA(StringRef Arch); +ARM::EndianKind parseArchEndian(StringRef Arch); +ARM::ProfileKind parseArchProfile(StringRef Arch); unsigned parseArchVersion(StringRef Arch); } // namespace AArch64 + +namespace X86 { + +// This should be kept in sync with libcc/compiler-rt as its included by clang +// as a proxy for what's in libgcc/compiler-rt. +enum ProcessorVendors : unsigned { + VENDOR_DUMMY, +#define X86_VENDOR(ENUM, STRING) \ + ENUM, +#include "llvm/Support/X86TargetParser.def" + VENDOR_OTHER +}; + +// This should be kept in sync with libcc/compiler-rt as its included by clang +// as a proxy for what's in libgcc/compiler-rt. +enum ProcessorTypes : unsigned { + CPU_TYPE_DUMMY, +#define X86_CPU_TYPE(ARCHNAME, ENUM) \ + ENUM, +#include "llvm/Support/X86TargetParser.def" + CPU_TYPE_MAX +}; + +// This should be kept in sync with libcc/compiler-rt as its included by clang +// as a proxy for what's in libgcc/compiler-rt. +enum ProcessorSubtypes : unsigned { + CPU_SUBTYPE_DUMMY, +#define X86_CPU_SUBTYPE(ARCHNAME, ENUM) \ + ENUM, +#include "llvm/Support/X86TargetParser.def" + CPU_SUBTYPE_MAX +}; + +// This should be kept in sync with libcc/compiler-rt as it should be used +// by clang as a proxy for what's in libgcc/compiler-rt. +enum ProcessorFeatures { +#define X86_FEATURE(VAL, ENUM) \ + ENUM = VAL, +#include "llvm/Support/X86TargetParser.def" + +}; + +} // namespace X86 + } // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/Support/TargetRegistry.h b/contrib/llvm/include/llvm/Support/TargetRegistry.h index 8454b27b6f04..bd096e2f74f6 100644 --- a/contrib/llvm/include/llvm/Support/TargetRegistry.h +++ b/contrib/llvm/include/llvm/Support/TargetRegistry.h @@ -67,15 +67,21 @@ MCStreamer *createAsmStreamer(MCContext &Ctx, MCAsmBackend *TAB, bool ShowInst); /// Takes ownership of \p TAB and \p CE. -MCStreamer *createELFStreamer(MCContext &Ctx, MCAsmBackend &TAB, - raw_pwrite_stream &OS, MCCodeEmitter *CE, +MCStreamer *createELFStreamer(MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&CE, bool RelaxAll); -MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB, - raw_pwrite_stream &OS, MCCodeEmitter *CE, +MCStreamer *createMachOStreamer(MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&CE, bool RelaxAll, bool DWARFMustBeAtTheEnd, bool LabelSections = false); -MCStreamer *createWasmStreamer(MCContext &Ctx, MCAsmBackend &TAB, - raw_pwrite_stream &OS, MCCodeEmitter *CE, +MCStreamer *createWasmStreamer(MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&CE, bool RelaxAll); MCRelocationInfo *createMCRelocationInfo(const Triple &TT, MCContext &Ctx); @@ -101,19 +107,16 @@ public: using MCAsmInfoCtorFnTy = MCAsmInfo *(*)(const MCRegisterInfo &MRI, const Triple &TT); - using MCAdjustCodeGenOptsFnTy = void (*)(const Triple &TT, Reloc::Model RM, - CodeModel::Model &CM); - using MCInstrInfoCtorFnTy = MCInstrInfo *(*)(); using MCInstrAnalysisCtorFnTy = MCInstrAnalysis *(*)(const MCInstrInfo *Info); using MCRegInfoCtorFnTy = MCRegisterInfo *(*)(const Triple &TT); using MCSubtargetInfoCtorFnTy = MCSubtargetInfo *(*)(const Triple &TT, StringRef CPU, StringRef Features); - using TargetMachineCtorTy = TargetMachine *(*)( - const Target &T, const Triple &TT, StringRef CPU, StringRef Features, - const TargetOptions &Options, Optional<Reloc::Model> RM, - CodeModel::Model CM, CodeGenOpt::Level OL); + using TargetMachineCtorTy = TargetMachine + *(*)(const Target &T, const Triple &TT, StringRef CPU, StringRef Features, + const TargetOptions &Options, Optional<Reloc::Model> RM, + Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT); // If it weren't for layering issues (this header is in llvm/Support, but // depends on MC?) this should take the Streamer by value rather than rvalue // reference. @@ -137,26 +140,26 @@ public: using MCCodeEmitterCtorTy = MCCodeEmitter *(*)(const MCInstrInfo &II, const MCRegisterInfo &MRI, MCContext &Ctx); - using ELFStreamerCtorTy = MCStreamer *(*)(const Triple &T, MCContext &Ctx, - MCAsmBackend &TAB, - raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, - bool RelaxAll); - using MachOStreamerCtorTy = MCStreamer *(*)(MCContext &Ctx, MCAsmBackend &TAB, - raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, - bool RelaxAll, - bool DWARFMustBeAtTheEnd); - using COFFStreamerCtorTy = MCStreamer *(*)(MCContext &Ctx, MCAsmBackend &TAB, - raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, - bool RelaxAll, - bool IncrementalLinkerCompatible); - using WasmStreamerCtorTy = MCStreamer *(*)(const Triple &T, MCContext &Ctx, - MCAsmBackend &TAB, - raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, - bool RelaxAll); + using ELFStreamerCtorTy = + MCStreamer *(*)(const Triple &T, MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll); + using MachOStreamerCtorTy = + MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll, + bool DWARFMustBeAtTheEnd); + using COFFStreamerCtorTy = + MCStreamer *(*)(MCContext &Ctx, std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll, + bool IncrementalLinkerCompatible); + using WasmStreamerCtorTy = + MCStreamer *(*)(const Triple &T, MCContext &Ctx, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&Emitter, bool RelaxAll); using NullTargetStreamerCtorTy = MCTargetStreamer *(*)(MCStreamer &S); using AsmTargetStreamerCtorTy = MCTargetStreamer *(*)( MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint, @@ -184,6 +187,10 @@ private: /// ShortDesc - A short description of the target. const char *ShortDesc; + /// BackendName - The name of the backend implementation. This must match the + /// name of the 'def X : Target ...' in TableGen. + const char *BackendName; + /// HasJIT - Whether this target supports the JIT. bool HasJIT; @@ -191,8 +198,6 @@ private: /// registered. MCAsmInfoCtorFnTy MCAsmInfoCtorFn; - MCAdjustCodeGenOptsFnTy MCAdjustCodeGenOptsFn; - /// MCInstrInfoCtorFn - Constructor function for this target's MCInstrInfo, /// if registered. MCInstrInfoCtorFnTy MCInstrInfoCtorFn; @@ -278,6 +283,9 @@ public: /// getShortDescription - Get a short description of the target. const char *getShortDescription() const { return ShortDesc; } + /// getBackendName - Get the backend name. + const char *getBackendName() const { return BackendName; } + /// @} /// @name Feature Predicates /// @{ @@ -312,12 +320,6 @@ public: return MCAsmInfoCtorFn(MRI, Triple(TheTriple)); } - void adjustCodeGenOpts(const Triple &TT, Reloc::Model RM, - CodeModel::Model &CM) const { - if (MCAdjustCodeGenOptsFn) - MCAdjustCodeGenOptsFn(TT, RM, CM); - } - /// createMCInstrInfo - Create a MCInstrInfo implementation. /// MCInstrInfo *createMCInstrInfo() const { @@ -365,15 +367,17 @@ public: /// feature set; it should always be provided. Generally this should be /// either the target triple from the module, or the target triple of the /// host if that does not exist. - TargetMachine * - createTargetMachine(StringRef TT, StringRef CPU, StringRef Features, - const TargetOptions &Options, Optional<Reloc::Model> RM, - CodeModel::Model CM = CodeModel::Default, - CodeGenOpt::Level OL = CodeGenOpt::Default) const { + TargetMachine *createTargetMachine(StringRef TT, StringRef CPU, + StringRef Features, + const TargetOptions &Options, + Optional<Reloc::Model> RM, + Optional<CodeModel::Model> CM = None, + CodeGenOpt::Level OL = CodeGenOpt::Default, + bool JIT = false) const { if (!TargetMachineCtorFn) return nullptr; return TargetMachineCtorFn(*this, Triple(TT), CPU, Features, Options, RM, - CM, OL); + CM, OL, JIT); } /// createMCAsmBackend - Create a target specific assembly parser. @@ -444,8 +448,9 @@ public: /// \param Emitter The target independent assembler object.Takes ownership. /// \param RelaxAll Relax all fixups? MCStreamer *createMCObjectStreamer(const Triple &T, MCContext &Ctx, - MCAsmBackend &TAB, raw_pwrite_stream &OS, - MCCodeEmitter *Emitter, + std::unique_ptr<MCAsmBackend> &&TAB, + raw_pwrite_stream &OS, + std::unique_ptr<MCCodeEmitter> &&Emitter, const MCSubtargetInfo &STI, bool RelaxAll, bool IncrementalLinkerCompatible, bool DWARFMustBeAtTheEnd) const { @@ -455,28 +460,32 @@ public: llvm_unreachable("Unknown object format"); case Triple::COFF: assert(T.isOSWindows() && "only Windows COFF is supported"); - S = COFFStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll, - IncrementalLinkerCompatible); + S = COFFStreamerCtorFn(Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll, IncrementalLinkerCompatible); break; case Triple::MachO: if (MachOStreamerCtorFn) - S = MachOStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll, - DWARFMustBeAtTheEnd); + S = MachOStreamerCtorFn(Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll, DWARFMustBeAtTheEnd); else - S = createMachOStreamer(Ctx, TAB, OS, Emitter, RelaxAll, - DWARFMustBeAtTheEnd); + S = createMachOStreamer(Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll, DWARFMustBeAtTheEnd); break; case Triple::ELF: if (ELFStreamerCtorFn) - S = ELFStreamerCtorFn(T, Ctx, TAB, OS, Emitter, RelaxAll); + S = ELFStreamerCtorFn(T, Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll); else - S = createELFStreamer(Ctx, TAB, OS, Emitter, RelaxAll); + S = createELFStreamer(Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll); break; case Triple::Wasm: if (WasmStreamerCtorFn) - S = WasmStreamerCtorFn(T, Ctx, TAB, OS, Emitter, RelaxAll); + S = WasmStreamerCtorFn(T, Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll); else - S = createWasmStreamer(Ctx, TAB, OS, Emitter, RelaxAll); + S = createWasmStreamer(Ctx, std::move(TAB), OS, std::move(Emitter), + RelaxAll); break; } if (ObjectTargetStreamerCtorFn) @@ -599,7 +608,7 @@ struct TargetRegistry { /// printRegisteredTargetsForVersion - Print the registered targets /// appropriately for inclusion in a tool's version output. - static void printRegisteredTargetsForVersion(); + static void printRegisteredTargetsForVersion(raw_ostream &OS); /// @name Registry Access /// @{ @@ -643,10 +652,15 @@ struct TargetRegistry { /// @param Name - The target name. This should be a static string. /// @param ShortDesc - A short target description. This should be a static /// string. + /// @param BackendName - The name of the backend. This should be a static + /// string that is the same for all targets that share a backend + /// implementation and must match the name used in the 'def X : Target ...' in + /// TableGen. /// @param ArchMatchFn - The arch match checking function for this target. /// @param HasJIT - Whether the target supports JIT code /// generation. static void RegisterTarget(Target &T, const char *Name, const char *ShortDesc, + const char *BackendName, Target::ArchMatchFnTy ArchMatchFn, bool HasJIT = false); @@ -663,11 +677,6 @@ struct TargetRegistry { T.MCAsmInfoCtorFn = Fn; } - static void registerMCAdjustCodeGenOpts(Target &T, - Target::MCAdjustCodeGenOptsFnTy Fn) { - T.MCAdjustCodeGenOptsFn = Fn; - } - /// RegisterMCInstrInfo - Register a MCInstrInfo implementation for the /// given target. /// @@ -881,13 +890,15 @@ struct TargetRegistry { /// } /// extern "C" void LLVMInitializeFooTargetInfo() { /// RegisterTarget<Triple::foo> X(getTheFooTarget(), "foo", "Foo -/// description"); +/// description", "Foo" /* Backend Name */); /// } template <Triple::ArchType TargetArchType = Triple::UnknownArch, bool HasJIT = false> struct RegisterTarget { - RegisterTarget(Target &T, const char *Name, const char *Desc) { - TargetRegistry::RegisterTarget(T, Name, Desc, &getArchMatch, HasJIT); + RegisterTarget(Target &T, const char *Name, const char *Desc, + const char *BackendName) { + TargetRegistry::RegisterTarget(T, Name, Desc, BackendName, &getArchMatch, + HasJIT); } static bool getArchMatch(Triple::ArchType Arch) { @@ -929,12 +940,6 @@ struct RegisterMCAsmInfoFn { } }; -struct RegisterMCAdjustCodeGenOptsFn { - RegisterMCAdjustCodeGenOptsFn(Target &T, Target::MCAdjustCodeGenOptsFnTy Fn) { - TargetRegistry::registerMCAdjustCodeGenOpts(T, Fn); - } -}; - /// RegisterMCInstrInfo - Helper template for registering a target instruction /// info implementation. This invokes the static "Create" method on the class /// to actually do the construction. Usage: @@ -1080,12 +1085,11 @@ template <class TargetMachineImpl> struct RegisterTargetMachine { } private: - static TargetMachine *Allocator(const Target &T, const Triple &TT, - StringRef CPU, StringRef FS, - const TargetOptions &Options, - Optional<Reloc::Model> RM, - CodeModel::Model CM, CodeGenOpt::Level OL) { - return new TargetMachineImpl(T, TT, CPU, FS, Options, RM, CM, OL); + static TargetMachine * + Allocator(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, + const TargetOptions &Options, Optional<Reloc::Model> RM, + Optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) { + return new TargetMachineImpl(T, TT, CPU, FS, Options, RM, CM, OL, JIT); } }; diff --git a/contrib/llvm/include/llvm/Support/ThreadPool.h b/contrib/llvm/include/llvm/Support/ThreadPool.h index 9ada946c6dae..fb8255900510 100644 --- a/contrib/llvm/include/llvm/Support/ThreadPool.h +++ b/contrib/llvm/include/llvm/Support/ThreadPool.h @@ -38,8 +38,8 @@ public: using TaskTy = std::function<void()>; using PackagedTaskTy = std::packaged_task<void()>; - /// Construct a pool with the number of core available on the system (or - /// whatever the value returned by std::thread::hardware_concurrency() is). + /// Construct a pool with the number of threads found by + /// hardware_concurrency(). ThreadPool(); /// Construct a pool of \p ThreadCount threads diff --git a/contrib/llvm/include/llvm/Support/Threading.h b/contrib/llvm/include/llvm/Support/Threading.h index 03963a24c107..6d813bccb93f 100644 --- a/contrib/llvm/include/llvm/Support/Threading.h +++ b/contrib/llvm/include/llvm/Support/Threading.h @@ -131,6 +131,14 @@ void llvm_execute_on_thread(void (*UserFn)(void *), void *UserData, /// Returns 1 when LLVM is configured with LLVM_ENABLE_THREADS=OFF unsigned heavyweight_hardware_concurrency(); + /// Get the number of threads that the current program can execute + /// concurrently. On some systems std::thread::hardware_concurrency() returns + /// the total number of cores, without taking affinity into consideration. + /// Returns 1 when LLVM is configured with LLVM_ENABLE_THREADS=OFF. + /// Fallback to std::thread::hardware_concurrency() if sched_getaffinity is + /// not available. + unsigned hardware_concurrency(); + /// \brief Return the current thread id, as used in various OS system calls. /// Note that not all platforms guarantee that the value returned will be /// unique across the entire system, so portable code should not assume diff --git a/contrib/llvm/include/llvm/Support/ToolOutputFile.h b/contrib/llvm/include/llvm/Support/ToolOutputFile.h index 1be26c2cb58b..b41ca5a6edaa 100644 --- a/contrib/llvm/include/llvm/Support/ToolOutputFile.h +++ b/contrib/llvm/include/llvm/Support/ToolOutputFile.h @@ -7,7 +7,7 @@ // //===----------------------------------------------------------------------===// // -// This file defines the tool_output_file class. +// This file defines the ToolOutputFile class. // //===----------------------------------------------------------------------===// @@ -21,9 +21,9 @@ namespace llvm { /// This class contains a raw_fd_ostream and adds a few extra features commonly /// needed for compiler-like tool output files: /// - The file is automatically deleted if the process is killed. -/// - The file is automatically deleted when the tool_output_file +/// - The file is automatically deleted when the ToolOutputFile /// object is destroyed unless the client calls keep(). -class tool_output_file { +class ToolOutputFile { /// This class is declared before the raw_fd_ostream so that it is constructed /// before the raw_fd_ostream is constructed and destructed after the /// raw_fd_ostream is destructed. It installs cleanups in its constructor and @@ -45,10 +45,10 @@ class tool_output_file { public: /// This constructor's arguments are passed to to raw_fd_ostream's /// constructor. - tool_output_file(StringRef Filename, std::error_code &EC, - sys::fs::OpenFlags Flags); + ToolOutputFile(StringRef Filename, std::error_code &EC, + sys::fs::OpenFlags Flags); - tool_output_file(StringRef Filename, int FD); + ToolOutputFile(StringRef Filename, int FD); /// Return the contained raw_fd_ostream. raw_fd_ostream &os() { return OS; } diff --git a/contrib/llvm/include/llvm/Support/X86TargetParser.def b/contrib/llvm/include/llvm/Support/X86TargetParser.def new file mode 100644 index 000000000000..5c8c576b1027 --- /dev/null +++ b/contrib/llvm/include/llvm/Support/X86TargetParser.def @@ -0,0 +1,155 @@ +//===- X86TargetParser.def - X86 target parsing defines ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides defines to build up the X86 target parser's logic. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef X86_VENDOR +#define X86_VENDOR(ENUM, STR) +#endif +X86_VENDOR(VENDOR_INTEL, "intel") +X86_VENDOR(VENDOR_AMD, "amd") +#undef X86_VENDOR + +// This macro is used to implement CPU types that have an alias. As of now +// there is only ever one alias. +#ifndef X86_CPU_TYPE_COMPAT_WITH_ALIAS +#define X86_CPU_TYPE_COMPAT_WITH_ALIAS(ARCHNAME, ENUM, STR, ALIAS) X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) +#endif + +// This macro is used for cpu types present in compiler-rt/libgcc. +#ifndef X86_CPU_TYPE_COMPAT +#define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) X86_CPU_TYPE(ARCHNAME, ENUM) +#endif + +#ifndef X86_CPU_TYPE +#define X86_CPU_TYPE(ARCHNAME, ENUM) +#endif +X86_CPU_TYPE_COMPAT_WITH_ALIAS("bonnell", INTEL_BONNELL, "bonnell", "atom") +X86_CPU_TYPE_COMPAT ("core2", INTEL_CORE2, "core2") +X86_CPU_TYPE_COMPAT ("nehalem", INTEL_COREI7, "corei7") +X86_CPU_TYPE_COMPAT_WITH_ALIAS("amdfam10", AMDFAM10H, "amdfam10h", "amdfam10") +X86_CPU_TYPE_COMPAT_WITH_ALIAS("bdver1", AMDFAM15H, "amdfam15h", "amdfam15") +X86_CPU_TYPE_COMPAT_WITH_ALIAS("silvermont", INTEL_SILVERMONT, "silvermont", "slm") +X86_CPU_TYPE_COMPAT ("knl", INTEL_KNL, "knl") +X86_CPU_TYPE_COMPAT ("btver1", AMD_BTVER1, "btver1") +X86_CPU_TYPE_COMPAT ("btver2", AMD_BTVER2, "btver2") +X86_CPU_TYPE_COMPAT ("znver1", AMDFAM17H, "amdfam17h") +X86_CPU_TYPE_COMPAT ("knm", INTEL_KNM, "knm") +// Entries below this are not in libgcc/compiler-rt. +X86_CPU_TYPE ("i386", INTEL_i386) +X86_CPU_TYPE ("i486", INTEL_i486) +X86_CPU_TYPE ("pentium", INTEL_PENTIUM) +X86_CPU_TYPE ("pentium-mmx", INTEL_PENTIUM_MMX) +X86_CPU_TYPE ("pentiumpro", INTEL_PENTIUM_PRO) +X86_CPU_TYPE ("pentium2", INTEL_PENTIUM_II) +X86_CPU_TYPE ("pentium3", INTEL_PENTIUM_III) +X86_CPU_TYPE ("pentium4", INTEL_PENTIUM_IV) +X86_CPU_TYPE ("pentium-m", INTEL_PENTIUM_M) +X86_CPU_TYPE ("yonah", INTEL_CORE_DUO) +X86_CPU_TYPE ("nocona", INTEL_NOCONA) +X86_CPU_TYPE ("prescott", INTEL_PRESCOTT) +X86_CPU_TYPE ("i486", AMD_i486) +X86_CPU_TYPE ("pentium", AMDPENTIUM) +X86_CPU_TYPE ("athlon", AMD_ATHLON) +X86_CPU_TYPE ("athlon-xp", AMD_ATHLON_XP) +X86_CPU_TYPE ("k8", AMD_K8) +X86_CPU_TYPE ("k8-sse3", AMD_K8SSE3) +X86_CPU_TYPE ("goldmont", INTEL_GOLDMONT) +#undef X86_CPU_TYPE_COMPAT_WITH_ALIAS +#undef X86_CPU_TYPE_COMPAT +#undef X86_CPU_TYPE + +// This macro is used for cpu subtypes present in compiler-rt/libgcc. +#ifndef X86_CPU_SUBTYPE_COMPAT +#define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) X86_CPU_SUBTYPE(ARCHNAME, ENUM) +#endif + +#ifndef X86_CPU_SUBTYPE +#define X86_CPU_SUBTYPE(ARCHNAME, ENUM) +#endif + +X86_CPU_SUBTYPE_COMPAT("nehalem", INTEL_COREI7_NEHALEM, "nehalem") +X86_CPU_SUBTYPE_COMPAT("westmere", INTEL_COREI7_WESTMERE, "westmere") +X86_CPU_SUBTYPE_COMPAT("sandybridge", INTEL_COREI7_SANDYBRIDGE, "sandybridge") +X86_CPU_SUBTYPE_COMPAT("amdfam10", AMDFAM10H_BARCELONA, "barcelona") +X86_CPU_SUBTYPE_COMPAT("amdfam10", AMDFAM10H_SHANGHAI, "shanghai") +X86_CPU_SUBTYPE_COMPAT("amdfam10", AMDFAM10H_ISTANBUL, "istanbul") +X86_CPU_SUBTYPE_COMPAT("bdver1", AMDFAM15H_BDVER1, "bdver1") +X86_CPU_SUBTYPE_COMPAT("bdver2", AMDFAM15H_BDVER2, "bdver2") +X86_CPU_SUBTYPE_COMPAT("bdver3", AMDFAM15H_BDVER3, "bdver3") +X86_CPU_SUBTYPE_COMPAT("bdver4", AMDFAM15H_BDVER4, "bdver4") +X86_CPU_SUBTYPE_COMPAT("znver1", AMDFAM17H_ZNVER1, "znver1") +X86_CPU_SUBTYPE_COMPAT("ivybridge", INTEL_COREI7_IVYBRIDGE, "ivybridge") +X86_CPU_SUBTYPE_COMPAT("haswell", INTEL_COREI7_HASWELL, "haswell") +X86_CPU_SUBTYPE_COMPAT("broadwell", INTEL_COREI7_BROADWELL, "broadwell") +X86_CPU_SUBTYPE_COMPAT("skylake", INTEL_COREI7_SKYLAKE, "skylake") +X86_CPU_SUBTYPE_COMPAT("skylake-avx512", INTEL_COREI7_SKYLAKE_AVX512, "skylake-avx512") +X86_CPU_SUBTYPE_COMPAT("cannonlake", INTEL_COREI7_CANNONLAKE, "cannonlake") +// Entries below this are not in libgcc/compiler-rt. +X86_CPU_SUBTYPE ("core2", INTEL_CORE2_65) +X86_CPU_SUBTYPE ("penryn", INTEL_CORE2_45) +X86_CPU_SUBTYPE ("k6", AMDPENTIUM_K6) +X86_CPU_SUBTYPE ("k6-2", AMDPENTIUM_K62) +X86_CPU_SUBTYPE ("k6-3", AMDPENTIUM_K63) +X86_CPU_SUBTYPE ("geode", AMDPENTIUM_GEODE) +#undef X86_CPU_SUBTYPE_COMPAT +#undef X86_CPU_SUBTYPE + + +// This macro is used for cpu types present in compiler-rt/libgcc. +#ifndef X86_FEATURE_COMPAT +#define X86_FEATURE_COMPAT(VAL, ENUM, STR) X86_FEATURE(VAL, ENUM) +#endif + +#ifndef X86_FEATURE +#define X86_FEATURE(VAL, ENUM) +#endif +X86_FEATURE_COMPAT( 0, FEATURE_CMOV, "cmov") +X86_FEATURE_COMPAT( 1, FEATURE_MMX, "mmx") +X86_FEATURE_COMPAT( 2, FEATURE_POPCNT, "popcnt") +X86_FEATURE_COMPAT( 3, FEATURE_SSE, "sse") +X86_FEATURE_COMPAT( 4, FEATURE_SSE2, "sse2") +X86_FEATURE_COMPAT( 5, FEATURE_SSE3, "sse3") +X86_FEATURE_COMPAT( 6, FEATURE_SSSE3, "ssse3") +X86_FEATURE_COMPAT( 7, FEATURE_SSE4_1, "sse4.1") +X86_FEATURE_COMPAT( 8, FEATURE_SSE4_2, "sse4.2") +X86_FEATURE_COMPAT( 9, FEATURE_AVX, "avx") +X86_FEATURE_COMPAT(10, FEATURE_AVX2, "avx2") +X86_FEATURE_COMPAT(11, FEATURE_SSE4_A, "sse4a") +X86_FEATURE_COMPAT(12, FEATURE_FMA4, "fma4") +X86_FEATURE_COMPAT(13, FEATURE_XOP, "xop") +X86_FEATURE_COMPAT(14, FEATURE_FMA, "fma") +X86_FEATURE_COMPAT(15, FEATURE_AVX512F, "avx512f") +X86_FEATURE_COMPAT(16, FEATURE_BMI, "bmi") +X86_FEATURE_COMPAT(17, FEATURE_BMI2, "bmi2") +X86_FEATURE_COMPAT(18, FEATURE_AES, "aes") +X86_FEATURE_COMPAT(19, FEATURE_PCLMUL, "pclmul") +X86_FEATURE_COMPAT(20, FEATURE_AVX512VL, "avx512vl") +X86_FEATURE_COMPAT(21, FEATURE_AVX512BW, "avx512bw") +X86_FEATURE_COMPAT(22, FEATURE_AVX512DQ, "avx512dq") +X86_FEATURE_COMPAT(23, FEATURE_AVX512CD, "avx512cd") +X86_FEATURE_COMPAT(24, FEATURE_AVX512ER, "avx512er") +X86_FEATURE_COMPAT(25, FEATURE_AVX512PF, "avx512pf") +X86_FEATURE_COMPAT(26, FEATURE_AVX512VBMI, "avx512vbmi") +X86_FEATURE_COMPAT(27, FEATURE_AVX512IFMA, "avx512ifma") +X86_FEATURE_COMPAT(28, FEATURE_AVX5124VNNIW, "avx5124vnniw") +X86_FEATURE_COMPAT(29, FEATURE_AVX5124FMAPS, "avx5124fmaps") +X86_FEATURE_COMPAT(30, FEATURE_AVX512VPOPCNTDQ, "avx512vpopcntdq") +// Features below here are not in libgcc/compiler-rt. +X86_FEATURE (32, FEATURE_MOVBE) +X86_FEATURE (33, FEATURE_ADX) +X86_FEATURE (34, FEATURE_EM64T) +X86_FEATURE (35, FEATURE_CLFLUSHOPT) +X86_FEATURE (36, FEATURE_SHA) +#undef X86_FEATURE_COMPAT +#undef X86_FEATURE diff --git a/contrib/llvm/include/llvm/Support/YAMLParser.h b/contrib/llvm/include/llvm/Support/YAMLParser.h index 549da3ccad51..c907a99ddb59 100644 --- a/contrib/llvm/include/llvm/Support/YAMLParser.h +++ b/contrib/llvm/include/llvm/Support/YAMLParser.h @@ -291,9 +291,11 @@ public: Node *getValue(); void skip() override { - getKey()->skip(); - if (Node *Val = getValue()) - Val->skip(); + if (Node *Key = getKey()) { + Key->skip(); + if (Node *Val = getValue()) + Val->skip(); + } } static bool classof(const Node *N) { @@ -572,13 +574,15 @@ public: document_iterator() = default; document_iterator(std::unique_ptr<Document> &D) : Doc(&D) {} - bool operator==(const document_iterator &Other) { + bool operator==(const document_iterator &Other) const { if (isAtEnd() || Other.isAtEnd()) return isAtEnd() && Other.isAtEnd(); return Doc == Other.Doc; } - bool operator!=(const document_iterator &Other) { return !(*this == Other); } + bool operator!=(const document_iterator &Other) const { + return !(*this == Other); + } document_iterator operator++() { assert(Doc && "incrementing iterator past the end."); diff --git a/contrib/llvm/include/llvm/Support/YAMLTraits.h b/contrib/llvm/include/llvm/Support/YAMLTraits.h index 71fdf47f1979..83b097a199d6 100644 --- a/contrib/llvm/include/llvm/Support/YAMLTraits.h +++ b/contrib/llvm/include/llvm/Support/YAMLTraits.h @@ -12,6 +12,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" @@ -117,6 +118,11 @@ struct ScalarBitSetTraits { // static void bitset(IO &io, T &value); }; +/// Describe which type of quotes should be used when quoting is necessary. +/// Some non-printable characters need to be double-quoted, while some others +/// are fine with simple-quoting, and some don't need any quoting. +enum class QuotingType { None, Single, Double }; + /// This class should be specialized by type that requires custom conversion /// to/from a yaml scalar. For example: /// @@ -131,7 +137,7 @@ struct ScalarBitSetTraits { /// // return empty string on success, or error string /// return StringRef(); /// } -/// static bool mustQuote(StringRef) { return true; } +/// static QuotingType mustQuote(StringRef) { return QuotingType::Single; } /// }; template<typename T> struct ScalarTraits { @@ -145,7 +151,7 @@ struct ScalarTraits { //static StringRef input(StringRef scalar, void *ctxt, T &value); // // Function to determine if the value should be quoted. - //static bool mustQuote(StringRef); + //static QuotingType mustQuote(StringRef); }; /// This class should be specialized by type that requires custom conversion @@ -270,7 +276,7 @@ struct has_ScalarTraits { using Signature_input = StringRef (*)(StringRef, void*, T&); using Signature_output = void (*)(const T&, void*, raw_ostream&); - using Signature_mustQuote = bool (*)(StringRef); + using Signature_mustQuote = QuotingType (*)(StringRef); template <typename U> static char test(SameType<Signature_input, &U::input> *, @@ -495,28 +501,66 @@ inline bool isBool(StringRef S) { S.equals("false") || S.equals("False") || S.equals("FALSE"); } -inline bool needsQuotes(StringRef S) { +// 5.1. Character Set +// The allowed character range explicitly excludes the C0 control block #x0-#x1F +// (except for TAB #x9, LF #xA, and CR #xD which are allowed), DEL #x7F, the C1 +// control block #x80-#x9F (except for NEL #x85 which is allowed), the surrogate +// block #xD800-#xDFFF, #xFFFE, and #xFFFF. +inline QuotingType needsQuotes(StringRef S) { if (S.empty()) - return true; + return QuotingType::Single; if (isspace(S.front()) || isspace(S.back())) - return true; + return QuotingType::Single; if (S.front() == ',') - return true; - - static const char ScalarSafeChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-/^., \t"; - if (S.find_first_not_of(ScalarSafeChars) != StringRef::npos) - return true; - + return QuotingType::Single; if (isNull(S)) - return true; + return QuotingType::Single; if (isBool(S)) - return true; + return QuotingType::Single; if (isNumeric(S)) - return true; + return QuotingType::Single; + + QuotingType MaxQuotingNeeded = QuotingType::None; + for (unsigned char C : S) { + // Alphanum is safe. + if (isAlnum(C)) + continue; + + switch (C) { + // Safe scalar characters. + case '_': + case '-': + case '/': + case '^': + case '.': + case ',': + case ' ': + // TAB (0x9), LF (0xA), CR (0xD) and NEL (0x85) are allowed. + case 0x9: + case 0xA: + case 0xD: + case 0x85: + continue; + // DEL (0x7F) are excluded from the allowed character range. + case 0x7F: + return QuotingType::Double; + default: { + // C0 control block (0x0 - 0x1F) is excluded from the allowed character + // range. + if (C <= 0x1F) + return QuotingType::Double; + // C1 control block (0x80 - 0x9F) is excluded from the allowed character + // range. + if (C >= 0x80 && C <= 0x9F) + return QuotingType::Double; + + // The character is not safe, at least simple quoting needed. + MaxQuotingNeeded = QuotingType::Single; + } + } + } - return false; + return MaxQuotingNeeded; } template <typename T, typename Context> @@ -581,7 +625,7 @@ public: virtual bool bitSetMatch(const char*, bool) = 0; virtual void endBitSetScalar() = 0; - virtual void scalarString(StringRef &, bool) = 0; + virtual void scalarString(StringRef &, QuotingType) = 0; virtual void blockScalarString(StringRef &) = 0; virtual void setError(const Twine &) = 0; @@ -911,91 +955,91 @@ template<> struct ScalarTraits<bool> { static void output(const bool &, void* , raw_ostream &); static StringRef input(StringRef, void *, bool &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<StringRef> { static void output(const StringRef &, void *, raw_ostream &); static StringRef input(StringRef, void *, StringRef &); - static bool mustQuote(StringRef S) { return needsQuotes(S); } + static QuotingType mustQuote(StringRef S) { return needsQuotes(S); } }; template<> struct ScalarTraits<std::string> { static void output(const std::string &, void *, raw_ostream &); static StringRef input(StringRef, void *, std::string &); - static bool mustQuote(StringRef S) { return needsQuotes(S); } + static QuotingType mustQuote(StringRef S) { return needsQuotes(S); } }; template<> struct ScalarTraits<uint8_t> { static void output(const uint8_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, uint8_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<uint16_t> { static void output(const uint16_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, uint16_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<uint32_t> { static void output(const uint32_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, uint32_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<uint64_t> { static void output(const uint64_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, uint64_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<int8_t> { static void output(const int8_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, int8_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<int16_t> { static void output(const int16_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, int16_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<int32_t> { static void output(const int32_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, int32_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<int64_t> { static void output(const int64_t &, void *, raw_ostream &); static StringRef input(StringRef, void *, int64_t &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<float> { static void output(const float &, void *, raw_ostream &); static StringRef input(StringRef, void *, float &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<double> { static void output(const double &, void *, raw_ostream &); static StringRef input(StringRef, void *, double &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; // For endian types, we just use the existing ScalarTraits for the underlying @@ -1019,7 +1063,7 @@ struct ScalarTraits<support::detail::packed_endian_specific_integral< return R; } - static bool mustQuote(StringRef Str) { + static QuotingType mustQuote(StringRef Str) { return ScalarTraits<value_type>::mustQuote(Str); } }; @@ -1148,7 +1192,7 @@ private: bool beginBitSetScalar(bool &) override; bool bitSetMatch(const char *, bool ) override; void endBitSetScalar() override; - void scalarString(StringRef &, bool) override; + void scalarString(StringRef &, QuotingType) override; void blockScalarString(StringRef &) override; void setError(const Twine &message) override; bool canElideEmptySequence() override; @@ -1293,7 +1337,7 @@ public: bool beginBitSetScalar(bool &) override; bool bitSetMatch(const char *, bool ) override; void endBitSetScalar() override; - void scalarString(StringRef &, bool) override; + void scalarString(StringRef &, QuotingType) override; void blockScalarString(StringRef &) override; void setError(const Twine &message) override; bool canElideEmptySequence() override; @@ -1371,28 +1415,28 @@ template<> struct ScalarTraits<Hex8> { static void output(const Hex8 &, void *, raw_ostream &); static StringRef input(StringRef, void *, Hex8 &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<Hex16> { static void output(const Hex16 &, void *, raw_ostream &); static StringRef input(StringRef, void *, Hex16 &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<Hex32> { static void output(const Hex32 &, void *, raw_ostream &); static StringRef input(StringRef, void *, Hex32 &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; template<> struct ScalarTraits<Hex64> { static void output(const Hex64 &, void *, raw_ostream &); static StringRef input(StringRef, void *, Hex64 &); - static bool mustQuote(StringRef) { return false; } + static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; // Define non-member operator>> so that Input can stream in a document list. @@ -1681,7 +1725,7 @@ template <typename T> struct StdMapStringCustomMappingTraitsImpl { template <> struct ScalarTraits<Type> { \ static void output(const Type &Value, void *ctx, raw_ostream &Out); \ static StringRef input(StringRef Scalar, void *ctxt, Type &Value); \ - static bool mustQuote(StringRef) { return MustQuote; } \ + static QuotingType mustQuote(StringRef) { return MustQuote; } \ }; \ } \ } diff --git a/contrib/llvm/include/llvm/Support/raw_ostream.h b/contrib/llvm/include/llvm/Support/raw_ostream.h index e644a5bda5ef..d11f5a837796 100644 --- a/contrib/llvm/include/llvm/Support/raw_ostream.h +++ b/contrib/llvm/include/llvm/Support/raw_ostream.h @@ -213,6 +213,10 @@ public: /// Output \p N in hexadecimal, without any prefix or padding. raw_ostream &write_hex(unsigned long long N); + /// Output a formatted UUID with dash separators. + using uuid_t = uint8_t[16]; + raw_ostream &write_uuid(const uuid_t UUID); + /// Output \p Str, turning '\\', '\t', '\n', '"', and anything that doesn't /// satisfy std::isprint into an escape sequence. raw_ostream &write_escaped(StringRef Str, bool UseHexEscapes = false); @@ -358,9 +362,7 @@ class raw_fd_ostream : public raw_pwrite_stream { int FD; bool ShouldClose; - /// Error This flag is true if an error of any kind has been detected. - /// - bool Error; + std::error_code EC; uint64_t pos; @@ -379,7 +381,7 @@ class raw_fd_ostream : public raw_pwrite_stream { size_t preferred_buffer_size() const override; /// Set the flag indicating that an output error has been encountered. - void error_detected() { Error = true; } + void error_detected(std::error_code EC) { this->EC = EC; } public: /// Open the specified file for writing. If an error occurs, information @@ -388,15 +390,14 @@ public: /// \p Flags allows optional flags to control how the file will be opened. /// /// As a special case, if Filename is "-", then the stream will use - /// STDOUT_FILENO instead of opening a file. Note that it will still consider - /// itself to own the file descriptor. In particular, it will close the - /// file descriptor when it is done (this is necessary to detect - /// output errors). + /// STDOUT_FILENO instead of opening a file. This will not close the stdout + /// descriptor. raw_fd_ostream(StringRef Filename, std::error_code &EC, sys::fs::OpenFlags Flags); /// FD is the file descriptor that this writes to. If ShouldClose is true, - /// this closes the file when the stream is destroyed. + /// this closes the file when the stream is destroyed. If FD is for stdout or + /// stderr, it will not be closed. raw_fd_ostream(int fd, bool shouldClose, bool unbuffered=false); ~raw_fd_ostream() override; @@ -421,13 +422,13 @@ public: bool has_colors() const override; + std::error_code error() const { return EC; } + /// Return the value of the flag in this raw_fd_ostream indicating whether an /// output error has been encountered. /// This doesn't implicitly flush any pending output. Also, it doesn't /// guarantee to detect all errors unless the stream has been closed. - bool has_error() const { - return Error; - } + bool has_error() const { return bool(EC); } /// Set the flag read by has_error() to false. If the error flag is set at the /// time when this raw_ostream's destructor is called, report_fatal_error is @@ -438,9 +439,7 @@ public: /// Unless explicitly silenced." /// - from The Zen of Python, by Tim Peters /// - void clear_error() { - Error = false; - } + void clear_error() { EC = std::error_code(); } }; /// This returns a reference to a raw_ostream for standard output. Use it like: diff --git a/contrib/llvm/include/llvm/TableGen/Error.h b/contrib/llvm/include/llvm/TableGen/Error.h index 3df658df8809..de4d3bf54782 100644 --- a/contrib/llvm/include/llvm/TableGen/Error.h +++ b/contrib/llvm/include/llvm/TableGen/Error.h @@ -19,6 +19,8 @@ namespace llvm { +void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg); + void PrintWarning(ArrayRef<SMLoc> WarningLoc, const Twine &Msg); void PrintWarning(const char *Loc, const Twine &Msg); void PrintWarning(const Twine &Msg); diff --git a/contrib/llvm/include/llvm/TableGen/Record.h b/contrib/llvm/include/llvm/TableGen/Record.h index fa9ca285bcde..55b4dfe2fa2f 100644 --- a/contrib/llvm/include/llvm/TableGen/Record.h +++ b/contrib/llvm/include/llvm/TableGen/Record.h @@ -1638,7 +1638,7 @@ struct LessRecordRegister { if (LHSNumParts != RHSNumParts) return LHSNumParts < RHSNumParts; - // We expect the registers to be of the form [_a-zA-z]+([0-9]*[_a-zA-Z]*)*. + // We expect the registers to be of the form [_a-zA-Z]+([0-9]*[_a-zA-Z]*)*. for (size_t I = 0, E = LHSNumParts; I < E; I+=2) { std::pair<bool, StringRef> LHSPart = LHSParts.getPart(I); std::pair<bool, StringRef> RHSPart = RHSParts.getPart(I); diff --git a/contrib/llvm/include/llvm/TableGen/StringMatcher.h b/contrib/llvm/include/llvm/TableGen/StringMatcher.h index 7c919ffec7b6..09d2092d43b0 100644 --- a/contrib/llvm/include/llvm/TableGen/StringMatcher.h +++ b/contrib/llvm/include/llvm/TableGen/StringMatcher.h @@ -43,11 +43,12 @@ public: const std::vector<StringPair> &matches, raw_ostream &os) : StrVariableName(strVariableName), Matches(matches), OS(os) {} - void Emit(unsigned Indent = 0) const; + void Emit(unsigned Indent = 0, bool IgnoreDuplicates = false) const; private: - bool EmitStringMatcherForChar(const std::vector<const StringPair*> &Matches, - unsigned CharNo, unsigned IndentCount) const; + bool EmitStringMatcherForChar(const std::vector<const StringPair *> &Matches, + unsigned CharNo, unsigned IndentCount, + bool IgnoreDuplicates) const; }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Target/GenericOpcodes.td b/contrib/llvm/include/llvm/Target/GenericOpcodes.td index e35bcb015d6a..28c90bf22767 100644 --- a/contrib/llvm/include/llvm/Target/GenericOpcodes.td +++ b/contrib/llvm/include/llvm/Target/GenericOpcodes.td @@ -16,9 +16,11 @@ // Unary ops. //------------------------------------------------------------------------------ +class GenericInstruction : StandardPseudoInstruction; + // Extend the underlying scalar type of an operation, leaving the high bits // unspecified. -def G_ANYEXT : Instruction { +def G_ANYEXT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; @@ -26,7 +28,7 @@ def G_ANYEXT : Instruction { // Sign extend the underlying scalar type of an operation, copying the sign bit // into the newly-created space. -def G_SEXT : Instruction { +def G_SEXT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; @@ -34,7 +36,7 @@ def G_SEXT : Instruction { // Zero extend the underlying scalar type of an operation, putting zero bits // into the newly-created space. -def G_ZEXT : Instruction { +def G_ZEXT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; @@ -43,68 +45,74 @@ def G_ZEXT : Instruction { // Truncate the underlying scalar type of an operation. This is equivalent to // G_EXTRACT for scalar types, but acts elementwise on vectors. -def G_TRUNC : Instruction { +def G_TRUNC : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_IMPLICIT_DEF : Instruction { +def G_IMPLICIT_DEF : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins); let hasSideEffects = 0; } -def G_FRAME_INDEX : Instruction { +def G_PHI : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins variable_ops); + let hasSideEffects = 0; +} + +def G_FRAME_INDEX : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$src2); let hasSideEffects = 0; } -def G_GLOBAL_VALUE : Instruction { +def G_GLOBAL_VALUE : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$src); let hasSideEffects = 0; } -def G_INTTOPTR : Instruction { +def G_INTTOPTR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_PTRTOINT : Instruction { +def G_PTRTOINT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_BITCAST : Instruction { +def G_BITCAST : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_CONSTANT : Instruction { +def G_CONSTANT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$imm); let hasSideEffects = 0; } -def G_FCONSTANT : Instruction { +def G_FCONSTANT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$imm); let hasSideEffects = 0; } -def G_VASTART : Instruction { +def G_VASTART : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$list); let hasSideEffects = 0; let mayStore = 1; } -def G_VAARG : Instruction { +def G_VAARG : GenericInstruction { let OutOperandList = (outs type0:$val); let InOperandList = (ins type1:$list, unknown:$align); let hasSideEffects = 0; @@ -112,12 +120,18 @@ def G_VAARG : Instruction { let mayStore = 1; } +def G_BSWAP : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + //------------------------------------------------------------------------------ // Binary ops. //------------------------------------------------------------------------------ // Generic addition. -def G_ADD : Instruction { +def G_ADD : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -125,7 +139,7 @@ def G_ADD : Instruction { } // Generic subtraction. -def G_SUB : Instruction { +def G_SUB : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -133,7 +147,7 @@ def G_SUB : Instruction { } // Generic multiplication. -def G_MUL : Instruction { +def G_MUL : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -141,7 +155,7 @@ def G_MUL : Instruction { } // Generic signed division. -def G_SDIV : Instruction { +def G_SDIV : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -149,7 +163,7 @@ def G_SDIV : Instruction { } // Generic unsigned division. -def G_UDIV : Instruction { +def G_UDIV : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -157,7 +171,7 @@ def G_UDIV : Instruction { } // Generic signed remainder. -def G_SREM : Instruction { +def G_SREM : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -165,7 +179,7 @@ def G_SREM : Instruction { } // Generic unsigned remainder. -def G_UREM : Instruction { +def G_UREM : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -173,7 +187,7 @@ def G_UREM : Instruction { } // Generic bitwise and. -def G_AND : Instruction { +def G_AND : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -181,7 +195,7 @@ def G_AND : Instruction { } // Generic bitwise or. -def G_OR : Instruction { +def G_OR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -189,7 +203,7 @@ def G_OR : Instruction { } // Generic bitwise xor. -def G_XOR : Instruction { +def G_XOR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -197,55 +211,55 @@ def G_XOR : Instruction { } // Generic left-shift. -def G_SHL : Instruction { +def G_SHL : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic logical right-shift. -def G_LSHR : Instruction { +def G_LSHR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic arithmetic right-shift. -def G_ASHR : Instruction { +def G_ASHR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic integer comparison. -def G_ICMP : Instruction { +def G_ICMP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$tst, type1:$src1, type1:$src2); let hasSideEffects = 0; } // Generic floating-point comparison. -def G_FCMP : Instruction { +def G_FCMP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$tst, type1:$src1, type1:$src2); let hasSideEffects = 0; } // Generic select -def G_SELECT : Instruction { +def G_SELECT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$tst, type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic pointer offset. -def G_GEP : Instruction { +def G_GEP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type1:$src2); let hasSideEffects = 0; } -def G_PTR_MASK : Instruction { +def G_PTR_MASK : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src, unknown:$bits); let hasSideEffects = 0; @@ -256,14 +270,14 @@ def G_PTR_MASK : Instruction { //------------------------------------------------------------------------------ // Generic unsigned addition consuming and producing a carry flag. -def G_UADDE : Instruction { +def G_UADDE : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2, type1:$carry_in); let hasSideEffects = 0; } // Generic signed addition producing a carry flag. -def G_SADDO : Instruction { +def G_SADDO : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -271,21 +285,21 @@ def G_SADDO : Instruction { } // Generic unsigned subtraction consuming and producing a carry flag. -def G_USUBE : Instruction { +def G_USUBE : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2, type1:$carry_in); let hasSideEffects = 0; } // Generic unsigned subtraction producing a carry flag. -def G_SSUBO : Instruction { +def G_SSUBO : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic unsigned multiplication producing a carry flag. -def G_UMULO : Instruction { +def G_UMULO : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -293,7 +307,7 @@ def G_UMULO : Instruction { } // Generic signed multiplication producing a carry flag. -def G_SMULO : Instruction { +def G_SMULO : GenericInstruction { let OutOperandList = (outs type0:$dst, type1:$carry_out); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -302,7 +316,7 @@ def G_SMULO : Instruction { // Multiply two numbers at twice the incoming bit width (unsigned) and return // the high half of the result. -def G_UMULH : Instruction { +def G_UMULH : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -311,7 +325,7 @@ def G_UMULH : Instruction { // Multiply two numbers at twice the incoming bit width (signed) and return // the high half of the result. -def G_SMULH : Instruction { +def G_SMULH : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -322,43 +336,43 @@ def G_SMULH : Instruction { // Floating Point Unary Ops. //------------------------------------------------------------------------------ -def G_FNEG : Instruction { +def G_FNEG : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src); let hasSideEffects = 0; } -def G_FPEXT : Instruction { +def G_FPEXT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_FPTRUNC : Instruction { +def G_FPTRUNC : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_FPTOSI : Instruction { +def G_FPTOSI : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_FPTOUI : Instruction { +def G_FPTOUI : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_SITOFP : Instruction { +def G_SITOFP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; } -def G_UITOFP : Instruction { +def G_UITOFP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src); let hasSideEffects = 0; @@ -369,7 +383,7 @@ def G_UITOFP : Instruction { //------------------------------------------------------------------------------ // Generic FP addition. -def G_FADD : Instruction { +def G_FADD : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -377,7 +391,7 @@ def G_FADD : Instruction { } // Generic FP subtraction. -def G_FSUB : Instruction { +def G_FSUB : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -385,7 +399,7 @@ def G_FSUB : Instruction { } // Generic FP multiplication. -def G_FMUL : Instruction { +def G_FMUL : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; @@ -394,7 +408,7 @@ def G_FMUL : Instruction { // Generic fused multiply-add instruction. // Behaves like llvm fma intrinsic ie src1 * src2 + src3 -def G_FMA : Instruction { +def G_FMA : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2, type0:$src3); let hasSideEffects = 0; @@ -402,49 +416,49 @@ def G_FMA : Instruction { } // Generic FP division. -def G_FDIV : Instruction { +def G_FDIV : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Generic FP remainder. -def G_FREM : Instruction { +def G_FREM : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Floating point exponentiation. -def G_FPOW : Instruction { +def G_FPOW : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type0:$src2); let hasSideEffects = 0; } // Floating point base-e exponential of a value. -def G_FEXP : Instruction { +def G_FEXP : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); let hasSideEffects = 0; } // Floating point base-2 exponential of a value. -def G_FEXP2 : Instruction { +def G_FEXP2 : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); let hasSideEffects = 0; } // Floating point base-2 logarithm of a value. -def G_FLOG : Instruction { +def G_FLOG : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); let hasSideEffects = 0; } // Floating point base-2 logarithm of a value. -def G_FLOG2 : Instruction { +def G_FLOG2 : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); let hasSideEffects = 0; @@ -455,29 +469,71 @@ def G_FLOG2 : Instruction { //------------------------------------------------------------------------------ // Generic load. Expects a MachineMemOperand in addition to explicit operands. -def G_LOAD : Instruction { +def G_LOAD : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type1:$addr); + let InOperandList = (ins ptype1:$addr); let hasSideEffects = 0; let mayLoad = 1; } // Generic store. Expects a MachineMemOperand in addition to explicit operands. -def G_STORE : Instruction { +def G_STORE : GenericInstruction { let OutOperandList = (outs); - let InOperandList = (ins type0:$src, type1:$addr); + let InOperandList = (ins type0:$src, ptype1:$addr); + let hasSideEffects = 0; + let mayStore = 1; +} + +// Generic atomic cmpxchg with internal success check. Expects a +// MachineMemOperand in addition to explicit operands. +def G_ATOMIC_CMPXCHG_WITH_SUCCESS : GenericInstruction { + let OutOperandList = (outs type0:$oldval, type1:$success); + let InOperandList = (ins type2:$addr, type0:$cmpval, type0:$newval); let hasSideEffects = 0; + let mayLoad = 1; let mayStore = 1; } +// Generic atomic cmpxchg. Expects a MachineMemOperand in addition to explicit +// operands. +def G_ATOMIC_CMPXCHG : GenericInstruction { + let OutOperandList = (outs type0:$oldval); + let InOperandList = (ins ptype1:$addr, type0:$cmpval, type0:$newval); + let hasSideEffects = 0; + let mayLoad = 1; + let mayStore = 1; +} + +// Generic atomicrmw. Expects a MachineMemOperand in addition to explicit +// operands. +class G_ATOMICRMW_OP : GenericInstruction { + let OutOperandList = (outs type0:$oldval); + let InOperandList = (ins ptype1:$addr, type0:$val); + let hasSideEffects = 0; + let mayLoad = 1; + let mayStore = 1; +} + +def G_ATOMICRMW_XCHG : G_ATOMICRMW_OP; +def G_ATOMICRMW_ADD : G_ATOMICRMW_OP; +def G_ATOMICRMW_SUB : G_ATOMICRMW_OP; +def G_ATOMICRMW_AND : G_ATOMICRMW_OP; +def G_ATOMICRMW_NAND : G_ATOMICRMW_OP; +def G_ATOMICRMW_OR : G_ATOMICRMW_OP; +def G_ATOMICRMW_XOR : G_ATOMICRMW_OP; +def G_ATOMICRMW_MAX : G_ATOMICRMW_OP; +def G_ATOMICRMW_MIN : G_ATOMICRMW_OP; +def G_ATOMICRMW_UMAX : G_ATOMICRMW_OP; +def G_ATOMICRMW_UMIN : G_ATOMICRMW_OP; + //------------------------------------------------------------------------------ // Variadic ops //------------------------------------------------------------------------------ -// Extract multiple registers specified size, starting from blocks given by -// indexes. This will almost certainly be mapped to sub-register COPYs after +// Extract a register of the specified size, starting from the block given by +// index. This will almost certainly be mapped to sub-register COPYs after // register banks have been selected. -def G_EXTRACT : Instruction { +def G_EXTRACT : GenericInstruction { let OutOperandList = (outs type0:$res); let InOperandList = (ins type1:$src, unknown:$offset); let hasSideEffects = 0; @@ -486,34 +542,35 @@ def G_EXTRACT : Instruction { // Extract multiple registers specified size, starting from blocks given by // indexes. This will almost certainly be mapped to sub-register COPYs after // register banks have been selected. -def G_UNMERGE_VALUES : Instruction { - let OutOperandList = (outs); - let InOperandList = (ins variable_ops); +def G_UNMERGE_VALUES : GenericInstruction { + let OutOperandList = (outs type0:$dst0, variable_ops); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } // Insert a smaller register into a larger one at the specified bit-index. -def G_INSERT : Instruction { +def G_INSERT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src, type1:$op, unknown:$offset); let hasSideEffects = 0; } -def G_MERGE_VALUES : Instruction { +/// Concatenate multiple registers of the same size into a wider register. +def G_MERGE_VALUES : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins variable_ops); + let InOperandList = (ins type1:$src0, variable_ops); let hasSideEffects = 0; } // Intrinsic without side effects. -def G_INTRINSIC : Instruction { +def G_INTRINSIC : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins unknown:$intrin, variable_ops); let hasSideEffects = 0; } // Intrinsic with side effects. -def G_INTRINSIC_W_SIDE_EFFECTS : Instruction { +def G_INTRINSIC_W_SIDE_EFFECTS : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins unknown:$intrin, variable_ops); let hasSideEffects = 1; @@ -526,7 +583,7 @@ def G_INTRINSIC_W_SIDE_EFFECTS : Instruction { //------------------------------------------------------------------------------ // Generic unconditional branch. -def G_BR : Instruction { +def G_BR : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins unknown:$src1); let hasSideEffects = 0; @@ -536,7 +593,7 @@ def G_BR : Instruction { } // Generic conditional branch. -def G_BRCOND : Instruction { +def G_BRCOND : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$tst, unknown:$truebb); let hasSideEffects = 0; @@ -545,7 +602,7 @@ def G_BRCOND : Instruction { } // Generic indirect branch. -def G_BRINDIRECT : Instruction { +def G_BRINDIRECT : GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$src1); let hasSideEffects = 0; @@ -558,21 +615,21 @@ def G_BRINDIRECT : Instruction { //------------------------------------------------------------------------------ // Generic insertelement. -def G_INSERT_VECTOR_ELT : Instruction { +def G_INSERT_VECTOR_ELT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src, type1:$elt, type2:$idx); let hasSideEffects = 0; } // Generic extractelement. -def G_EXTRACT_VECTOR_ELT : Instruction { +def G_EXTRACT_VECTOR_ELT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src, type2:$idx); let hasSideEffects = 0; } // Generic shufflevector. -def G_SHUFFLE_VECTOR: Instruction { +def G_SHUFFLE_VECTOR: GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$v1, type1:$v2, type2:$mask); let hasSideEffects = 0; diff --git a/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 50de41fd1320..575f228cd773 100644 --- a/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/contrib/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -23,6 +23,11 @@ class GINodeEquiv<Instruction i, SDNode node> { Instruction I = i; SDNode Node = node; + + // SelectionDAG has separate nodes for atomic and non-atomic memory operations + // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel + // stores this information in the MachineMemoryOperand. + bit CheckMMOIsNonAtomic = 0; } // These are defined in the same order as the G_* instructions. @@ -33,8 +38,8 @@ def : GINodeEquiv<G_TRUNC, trunc>; def : GINodeEquiv<G_BITCAST, bitconvert>; // G_INTTOPTR - SelectionDAG has no equivalent. // G_PTRTOINT - SelectionDAG has no equivalent. -// G_CONSTANT - Not needed since constants aren't operators. -// G_FCONSTANT - Not needed since constants aren't operators. +def : GINodeEquiv<G_CONSTANT, imm>; +def : GINodeEquiv<G_FCONSTANT, fpimm>; def : GINodeEquiv<G_ADD, add>; def : GINodeEquiv<G_SUB, sub>; def : GINodeEquiv<G_MUL, mul>; @@ -51,7 +56,7 @@ def : GINodeEquiv<G_ASHR, sra>; def : GINodeEquiv<G_SELECT, select>; def : GINodeEquiv<G_FNEG, fneg>; def : GINodeEquiv<G_FPEXT, fpextend>; -def : GINodeEquiv<G_FPTRUNC, ftrunc>; +def : GINodeEquiv<G_FPTRUNC, fpround>; def : GINodeEquiv<G_FPTOSI, fp_to_sint>; def : GINodeEquiv<G_FPTOUI, fp_to_uint>; def : GINodeEquiv<G_SITOFP, sint_to_fp>; @@ -66,7 +71,41 @@ def : GINodeEquiv<G_FPOW, fpow>; def : GINodeEquiv<G_FEXP2, fexp2>; def : GINodeEquiv<G_FLOG2, flog2>; def : GINodeEquiv<G_INTRINSIC, intrinsic_wo_chain>; +// ISD::INTRINSIC_VOID can also be handled with G_INTRINSIC_W_SIDE_EFFECTS. +def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_void>; +def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>; def : GINodeEquiv<G_BR, br>; +def : GINodeEquiv<G_BSWAP, bswap>; + +// Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some +// complications that tablegen must take care of. For example, Predicates such +// as isSignExtLoad require that this is not a perfect 1:1 mapping since a +// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. Additionally, +// G_LOAD handles both atomic and non-atomic loads where as SelectionDAG had +// separate nodes for them. This GINodeEquiv maps the non-atomic loads to +// G_LOAD with a non-atomic MachineMemOperand. +def : GINodeEquiv<G_LOAD, ld> { let CheckMMOIsNonAtomic = 1; } +// Broadly speaking G_STORE is equivalent to ISD::STORE but there are some +// complications that tablegen must take care of. For example, predicates such +// as isTruncStore require that this is not a perfect 1:1 mapping since a +// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel. Additionally, +// G_STORE handles both atomic and non-atomic stores where as SelectionDAG had +// separate nodes for them. This GINodeEquiv maps the non-atomic stores to +// G_STORE with a non-atomic MachineMemOperand. +def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = 1; } + +def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>; +def : GINodeEquiv<G_ATOMICRMW_XCHG, atomic_swap>; +def : GINodeEquiv<G_ATOMICRMW_ADD, atomic_load_add>; +def : GINodeEquiv<G_ATOMICRMW_SUB, atomic_load_sub>; +def : GINodeEquiv<G_ATOMICRMW_AND, atomic_load_and>; +def : GINodeEquiv<G_ATOMICRMW_NAND, atomic_load_nand>; +def : GINodeEquiv<G_ATOMICRMW_OR, atomic_load_or>; +def : GINodeEquiv<G_ATOMICRMW_XOR, atomic_load_xor>; +def : GINodeEquiv<G_ATOMICRMW_MIN, atomic_load_min>; +def : GINodeEquiv<G_ATOMICRMW_MAX, atomic_load_max>; +def : GINodeEquiv<G_ATOMICRMW_UMIN, atomic_load_umin>; +def : GINodeEquiv<G_ATOMICRMW_UMAX, atomic_load_umax>; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. diff --git a/contrib/llvm/include/llvm/Target/Target.td b/contrib/llvm/include/llvm/Target/Target.td index 6f44292c47ed..82a3be5e63d4 100644 --- a/contrib/llvm/include/llvm/Target/Target.td +++ b/contrib/llvm/include/llvm/Target/Target.td @@ -21,6 +21,56 @@ include "llvm/IR/Intrinsics.td" class RegisterClass; // Forward def +class HwMode<string FS> { + // A string representing subtarget features that turn on this HW mode. + // For example, "+feat1,-feat2" will indicate that the mode is active + // when "feat1" is enabled and "feat2" is disabled at the same time. + // Any other features are not checked. + // When multiple modes are used, they should be mutually exclusive, + // otherwise the results are unpredictable. + string Features = FS; +} + +// A special mode recognized by tablegen. This mode is considered active +// when no other mode is active. For targets that do not use specific hw +// modes, this is the only mode. +def DefaultMode : HwMode<"">; + +// A class used to associate objects with HW modes. It is only intended to +// be used as a base class, where the derived class should contain a member +// "Objects", which is a list of the same length as the list of modes. +// The n-th element on the Objects list will be associated with the n-th +// element on the Modes list. +class HwModeSelect<list<HwMode> Ms> { + list<HwMode> Modes = Ms; +} + +// A common class that implements a counterpart of ValueType, which is +// dependent on a HW mode. This class inherits from ValueType itself, +// which makes it possible to use objects of this class where ValueType +// objects could be used. This is specifically applicable to selection +// patterns. +class ValueTypeByHwMode<list<HwMode> Ms, list<ValueType> Ts> + : HwModeSelect<Ms>, ValueType<0, 0> { + // The length of this list must be the same as the length of Ms. + list<ValueType> Objects = Ts; +} + +// A class representing the register size, spill size and spill alignment +// in bits of a register. +class RegInfo<int RS, int SS, int SA> { + int RegSize = RS; // Register size in bits. + int SpillSize = SS; // Spill slot size in bits. + int SpillAlignment = SA; // Spill slot alignment in bits. +} + +// The register size/alignment information, parameterized by a HW mode. +class RegInfoByHwMode<list<HwMode> Ms = [], list<RegInfo> Ts = []> + : HwModeSelect<Ms> { + // The length of this list must be the same as the length of Ms. + list<RegInfo> Objects = Ts; +} + // SubRegIndex - Use instances of SubRegIndex to identify subregisters. class SubRegIndex<int size, int offset = 0> { string Namespace = ""; @@ -156,6 +206,9 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment, : DAGOperand { string Namespace = namespace; + // The register size/alignment information, parameterized by a HW mode. + RegInfoByHwMode RegInfos; + // RegType - Specify the list ValueType of the registers in this register // class. Note that all registers in a register class must have the same // ValueTypes. This is a list because some targets permit storing different @@ -216,6 +269,21 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment, // useful as it is sometimes beneficial to assign registers to highly // constrained classes first. The value has to be in the range [0,63]. int AllocationPriority = 0; + + // The diagnostic type to present when referencing this operand in a match + // failure error message. If this is empty, the default Match_InvalidOperand + // diagnostic type will be used. If this is "<name>", a Match_<name> enum + // value will be generated and used for this operand type. The target + // assembly parser is responsible for converting this into a user-facing + // diagnostic message. + string DiagnosticType = ""; + + // A diagnostic message to emit when an invalid value is provided for this + // register class when it is being used an an assembly operand. If this is + // non-empty, an anonymous diagnostic type enum value will be generated, and + // the assembly matcher will provide a function to map from diagnostic types + // to message strings. + string DiagnosticString = ""; } // The memberList in a RegisterClass is a dag of set operations. TableGen @@ -624,6 +692,10 @@ class AsmOperandClass { // diagnostic. The target AsmParser maps these codes to text. string DiagnosticType = ""; + /// A diagnostic message to emit when an invalid value is provided for this + /// operand. + string DiagnosticString = ""; + /// Set to 1 if this operand is optional and not always required. Typically, /// the AsmParser will emit an error when it finishes parsing an /// instruction if it hasn't matched all the operands yet. However, this @@ -696,6 +768,12 @@ class RegisterOperand<RegisterClass regclass, string pm = "printOperand"> AsmOperandClass ParserMatchClass; string OperandType = "OPERAND_REGISTER"; + + // When referenced in the result of a CodeGen pattern, GlobalISel will + // normally copy the matched operand to the result. When this is set, it will + // emit a special copy that will replace zero-immediates with the specified + // zero-register. + Register GIZeroRegister = ?; } let OperandType = "OPERAND_IMMEDIATE" in { @@ -714,6 +792,7 @@ def f64imm : Operand<f64>; // have the same LLT). class TypedOperand<string Ty> : Operand<untyped> { let OperandType = Ty; + bit IsPointer = 0; } def type0 : TypedOperand<"OPERAND_GENERIC_0">; @@ -723,6 +802,15 @@ def type3 : TypedOperand<"OPERAND_GENERIC_3">; def type4 : TypedOperand<"OPERAND_GENERIC_4">; def type5 : TypedOperand<"OPERAND_GENERIC_5">; +let IsPointer = 1 in { + def ptype0 : TypedOperand<"OPERAND_GENERIC_0">; + def ptype1 : TypedOperand<"OPERAND_GENERIC_1">; + def ptype2 : TypedOperand<"OPERAND_GENERIC_2">; + def ptype3 : TypedOperand<"OPERAND_GENERIC_3">; + def ptype4 : TypedOperand<"OPERAND_GENERIC_4">; + def ptype5 : TypedOperand<"OPERAND_GENERIC_5">; +} + /// zero_reg definition - Special node to stand for the zero register. /// def zero_reg; @@ -806,60 +894,81 @@ class InstrInfo { // Standard Pseudo Instructions. // This list must match TargetOpcodes.h and CodeGenTarget.cpp. // Only these instructions are allowed in the TargetOpcode namespace. -let isCodeGenOnly = 1, isPseudo = 1, hasNoSchedulingInfo = 1, - Namespace = "TargetOpcode" in { -def PHI : Instruction { +// Ensure mayLoad and mayStore have a default value, so as not to break +// targets that set guessInstructionProperties=0. Any local definition of +// mayLoad/mayStore takes precedence over these default values. +class StandardPseudoInstruction : Instruction { + let mayLoad = 0; + let mayStore = 0; + let isCodeGenOnly = 1; + let isPseudo = 1; + let hasNoSchedulingInfo = 1; + let Namespace = "TargetOpcode"; +} +def PHI : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let AsmString = "PHINODE"; + let hasSideEffects = 0; } -def INLINEASM : Instruction { +def INLINEASM : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = ""; let hasSideEffects = 0; // Note side effect is encoded in an operand. } -def CFI_INSTRUCTION : Instruction { +def CFI_INSTRUCTION : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i32imm:$id); + let AsmString = ""; + let hasCtrlDep = 1; + let hasSideEffects = 0; + let isNotDuplicable = 1; +} +def EH_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = ""; let hasCtrlDep = 1; + let hasSideEffects = 0; let isNotDuplicable = 1; } -def EH_LABEL : Instruction { +def GC_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = ""; let hasCtrlDep = 1; + let hasSideEffects = 0; let isNotDuplicable = 1; } -def GC_LABEL : Instruction { +def ANNOTATION_LABEL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = ""; let hasCtrlDep = 1; + let hasSideEffects = 0; let isNotDuplicable = 1; } -def KILL : Instruction { +def KILL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = ""; let hasSideEffects = 0; } -def EXTRACT_SUBREG : Instruction { +def EXTRACT_SUBREG : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$supersrc, i32imm:$subidx); let AsmString = ""; let hasSideEffects = 0; } -def INSERT_SUBREG : Instruction { +def INSERT_SUBREG : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$supersrc, unknown:$subsrc, i32imm:$subidx); let AsmString = ""; let hasSideEffects = 0; let Constraints = "$supersrc = $dst"; } -def IMPLICIT_DEF : Instruction { +def IMPLICIT_DEF : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins); let AsmString = ""; @@ -867,33 +976,33 @@ def IMPLICIT_DEF : Instruction { let isReMaterializable = 1; let isAsCheapAsAMove = 1; } -def SUBREG_TO_REG : Instruction { +def SUBREG_TO_REG : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$implsrc, unknown:$subsrc, i32imm:$subidx); let AsmString = ""; let hasSideEffects = 0; } -def COPY_TO_REGCLASS : Instruction { +def COPY_TO_REGCLASS : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$src, i32imm:$regclass); let AsmString = ""; let hasSideEffects = 0; let isAsCheapAsAMove = 1; } -def DBG_VALUE : Instruction { +def DBG_VALUE : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "DBG_VALUE"; let hasSideEffects = 0; } -def REG_SEQUENCE : Instruction { +def REG_SEQUENCE : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$supersrc, variable_ops); let AsmString = ""; let hasSideEffects = 0; let isAsCheapAsAMove = 1; } -def COPY : Instruction { +def COPY : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins unknown:$src); let AsmString = ""; @@ -901,39 +1010,42 @@ def COPY : Instruction { let isAsCheapAsAMove = 1; let hasNoSchedulingInfo = 0; } -def BUNDLE : Instruction { +def BUNDLE : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "BUNDLE"; + let hasSideEffects = 1; } -def LIFETIME_START : Instruction { +def LIFETIME_START : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = "LIFETIME_START"; let hasSideEffects = 0; } -def LIFETIME_END : Instruction { +def LIFETIME_END : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); let AsmString = "LIFETIME_END"; let hasSideEffects = 0; } -def STACKMAP : Instruction { +def STACKMAP : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i64imm:$id, i32imm:$nbytes, variable_ops); + let hasSideEffects = 1; let isCall = 1; let mayLoad = 1; let usesCustomInserter = 1; } -def PATCHPOINT : Instruction { +def PATCHPOINT : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins i64imm:$id, i32imm:$nbytes, unknown:$callee, i32imm:$nargs, i32imm:$cc, variable_ops); + let hasSideEffects = 1; let isCall = 1; let mayLoad = 1; let usesCustomInserter = 1; } -def STATEPOINT : Instruction { +def STATEPOINT : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let usesCustomInserter = 1; @@ -942,7 +1054,7 @@ def STATEPOINT : Instruction { let hasSideEffects = 1; let isCall = 1; } -def LOAD_STACK_GUARD : Instruction { +def LOAD_STACK_GUARD : StandardPseudoInstruction { let OutOperandList = (outs ptr_rc:$dst); let InOperandList = (ins); let mayLoad = 1; @@ -950,7 +1062,7 @@ def LOAD_STACK_GUARD : Instruction { let hasSideEffects = 0; bit isPseudo = 1; } -def LOCAL_ESCAPE : Instruction { +def LOCAL_ESCAPE : StandardPseudoInstruction { // This instruction is really just a label. It has to be part of the chain so // that it doesn't get dropped from the DAG, but it produces nothing and has // no side effects. @@ -959,16 +1071,17 @@ def LOCAL_ESCAPE : Instruction { let hasSideEffects = 0; let hasCtrlDep = 1; } -def FAULTING_OP : Instruction { +def FAULTING_OP : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let usesCustomInserter = 1; + let hasSideEffects = 1; let mayLoad = 1; let mayStore = 1; let isTerminator = 1; let isBranch = 1; } -def PATCHABLE_OP : Instruction { +def PATCHABLE_OP : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let usesCustomInserter = 1; @@ -976,22 +1089,23 @@ def PATCHABLE_OP : Instruction { let mayStore = 1; let hasSideEffects = 1; } -def PATCHABLE_FUNCTION_ENTER : Instruction { +def PATCHABLE_FUNCTION_ENTER : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins); let AsmString = "# XRay Function Enter."; let usesCustomInserter = 1; let hasSideEffects = 0; } -def PATCHABLE_RET : Instruction { +def PATCHABLE_RET : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let AsmString = "# XRay Function Patchable RET."; let usesCustomInserter = 1; let hasSideEffects = 1; + let isTerminator = 1; let isReturn = 1; } -def PATCHABLE_FUNCTION_EXIT : Instruction { +def PATCHABLE_FUNCTION_EXIT : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins); let AsmString = "# XRay Function Exit."; @@ -999,7 +1113,7 @@ def PATCHABLE_FUNCTION_EXIT : Instruction { let hasSideEffects = 0; // FIXME: is this correct? let isReturn = 0; // Original return instruction will follow } -def PATCHABLE_TAIL_CALL : Instruction { +def PATCHABLE_TAIL_CALL : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let AsmString = "# XRay Tail Call Exit."; @@ -1007,7 +1121,7 @@ def PATCHABLE_TAIL_CALL : Instruction { let hasSideEffects = 1; let isReturn = 1; } -def PATCHABLE_EVENT_CALL : Instruction { +def PATCHABLE_EVENT_CALL : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins ptr_rc:$event, i8imm:$size); let AsmString = "# XRay Custom Event Log."; @@ -1017,7 +1131,7 @@ def PATCHABLE_EVENT_CALL : Instruction { let mayStore = 1; let hasSideEffects = 1; } -def FENTRY_CALL : Instruction { +def FENTRY_CALL : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); let AsmString = "# FEntry call"; @@ -1030,8 +1144,6 @@ def FENTRY_CALL : Instruction { // Generic opcodes used in GlobalISel. include "llvm/Target/GenericOpcodes.td" -} - //===----------------------------------------------------------------------===// // AsmParser - This class can be implemented by targets that wish to implement // .s file parsing. @@ -1062,9 +1174,25 @@ class AsmParser { // several registers share the same alias (i.e. not a 1:1 mapping). bit ShouldEmitMatchRegisterAltName = 0; + // Set to true if MatchRegisterName and MatchRegisterAltName functions + // should be generated even if there are duplicate register names. The + // target is responsible for coercing aliased registers as necessary + // (e.g. in validateTargetOperandClass), and there are no guarantees about + // which numeric register identifier will be returned in the case of + // multiple matches. + bit AllowDuplicateRegisterNames = 0; + // HasMnemonicFirst - Set to false if target instructions don't always // start with a mnemonic as the first token. bit HasMnemonicFirst = 1; + + // ReportMultipleNearMisses - + // When 0, the assembly matcher reports an error for one encoding or operand + // that did not match the parsed instruction. + // When 1, the assmebly matcher returns a list of encodings that were close + // to matching the parsed instruction, so to allow more detailed error + // messages. + bit ReportMultipleNearMisses = 0; } def DefaultAsmParser : AsmParser; diff --git a/contrib/llvm/include/llvm/Target/TargetMachine.h b/contrib/llvm/include/llvm/Target/TargetMachine.h index 933c6c87b0be..5421b22462ae 100644 --- a/contrib/llvm/include/llvm/Target/TargetMachine.h +++ b/contrib/llvm/include/llvm/Target/TargetMachine.h @@ -25,6 +25,7 @@ namespace llvm { class GlobalValue; +class MachineModuleInfo; class Mangler; class MCAsmInfo; class MCContext; @@ -77,7 +78,7 @@ protected: // Can only create subclasses. std::string TargetFS; Reloc::Model RM = Reloc::Static; - CodeModel::Model CMModel = CodeModel::Default; + CodeModel::Model CMModel = CodeModel::Small; CodeGenOpt::Level OptLevel = CodeGenOpt::Default; /// Contains target specific asm information. @@ -222,11 +223,12 @@ public: /// emitted. Typically this will involve several steps of code generation. /// This method should return true if emission of this file type is not /// supported, or false on success. - virtual bool addPassesToEmitFile( - PassManagerBase &, raw_pwrite_stream &, CodeGenFileType, - bool /*DisableVerify*/ = true, AnalysisID /*StartBefore*/ = nullptr, - AnalysisID /*StartAfter*/ = nullptr, AnalysisID /*StopBefore*/ = nullptr, - AnalysisID /*StopAfter*/ = nullptr) { + /// \p MMI is an optional parameter that, if set to non-nullptr, + /// will be used to set the MachineModuloInfo for this PM. + virtual bool addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &, + CodeGenFileType, + bool /*DisableVerify*/ = true, + MachineModuleInfo *MMI = nullptr) { return true; } @@ -257,6 +259,12 @@ public: /// PEI. If false (virtual-register machines), then callee-save register /// spilling and scavenging are not needed or used. virtual bool usesPhysRegsForPEI() const { return true; } + + /// True if the target wants to use interprocedural register allocation by + /// default. The -enable-ipra flag can be used to override this. + virtual bool useIPRA() const { + return false; + } }; /// This class describes a target machine that is implemented with the LLVM @@ -270,6 +278,7 @@ protected: // Can only create subclasses. CodeModel::Model CM, CodeGenOpt::Level OL); void initAsmInfo(); + public: /// \brief Get a TargetIRAnalysis implementation for the target. /// @@ -283,11 +292,11 @@ public: /// Add passes to the specified pass manager to get the specified file /// emitted. Typically this will involve several steps of code generation. - bool addPassesToEmitFile( - PassManagerBase &PM, raw_pwrite_stream &Out, CodeGenFileType FileType, - bool DisableVerify = true, AnalysisID StartBefore = nullptr, - AnalysisID StartAfter = nullptr, AnalysisID StopBefore = nullptr, - AnalysisID StopAfter = nullptr) override; + /// \p MMI is an optional parameter that, if set to non-nullptr, + /// will be used to set the MachineModuloInfofor this PM. + bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out, + CodeGenFileType FileType, bool DisableVerify = true, + MachineModuleInfo *MMI = nullptr) override; /// Add passes to the specified pass manager to get machine code emitted with /// the MCJIT. This method returns true if machine code is not supported. It diff --git a/contrib/llvm/include/llvm/Target/TargetOptions.h b/contrib/llvm/include/llvm/Target/TargetOptions.h index 5c2063880f8b..70fac7833f32 100644 --- a/contrib/llvm/include/llvm/Target/TargetOptions.h +++ b/contrib/llvm/include/llvm/Target/TargetOptions.h @@ -108,7 +108,7 @@ namespace llvm { DisableIntegratedAS(false), RelaxELFRelocations(false), FunctionSections(false), DataSections(false), UniqueSectionNames(true), TrapUnreachable(false), EmulatedTLS(false), - EnableIPRA(false) {} + EnableIPRA(false), EmitStackSizeSection(false) {} /// PrintMachineCode - This flag is enabled when the -print-machineinstrs /// option is specified on the command line, and should enable debugging @@ -216,6 +216,9 @@ namespace llvm { /// This flag enables InterProcedural Register Allocation (IPRA). unsigned EnableIPRA : 1; + /// Emit section containing metadata on function stack sizes. + unsigned EmitStackSizeSection : 1; + /// FloatABIType - This setting is set by -float-abi=xxx option is specfied /// on the command line. This setting may either be Default, Soft, or Hard. /// Default selects the target's default behavior. Soft selects the ABI for diff --git a/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td b/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td index 9ed614ccee17..06caa21d288c 100644 --- a/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/contrib/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -132,7 +132,7 @@ def SDTFPSignOp : SDTypeProfile<1, 2, [ // fcopysign. def SDTFPTernaryOp : SDTypeProfile<1, 3, [ // fmadd, fnmsub, etc. SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisFP<0> ]>; -def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // ctlz +def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // ctlz, cttz SDTCisSameAs<0, 1>, SDTCisInt<0> ]>; def SDTIntExtendOp : SDTypeProfile<1, 1, [ // sext, zext, anyext @@ -649,6 +649,60 @@ class PatFrag<dag ops, dag frag, code pred = [{}], code PredicateCode = pred; code ImmediateCode = [{}]; SDNodeXForm OperandTransform = xform; + + // Define a few pre-packaged predicates. This helps GlobalISel import + // existing rules from SelectionDAG for many common cases. + // They will be tested prior to the code in pred and must not be used in + // ImmLeaf and its subclasses. + + // Is the desired pre-packaged predicate for a load? + bit IsLoad = ?; + // Is the desired pre-packaged predicate for a store? + bit IsStore = ?; + // Is the desired pre-packaged predicate for an atomic? + bit IsAtomic = ?; + + // cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; + // cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; + bit IsUnindexed = ?; + + // cast<LoadSDNode>(N)->getExtensionType() != ISD::NON_EXTLOAD + bit IsNonExtLoad = ?; + // cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; + bit IsAnyExtLoad = ?; + // cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; + bit IsSignExtLoad = ?; + // cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; + bit IsZeroExtLoad = ?; + // !cast<StoreSDNode>(N)->isTruncatingStore(); + // cast<StoreSDNode>(N)->isTruncatingStore(); + bit IsTruncStore = ?; + + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Monotonic + bit IsAtomicOrderingMonotonic = ?; + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Acquire + bit IsAtomicOrderingAcquire = ?; + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Release + bit IsAtomicOrderingRelease = ?; + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::AcquireRelease + bit IsAtomicOrderingAcquireRelease = ?; + // cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent + bit IsAtomicOrderingSequentiallyConsistent = ?; + + // isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering()) + // !isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering()) + bit IsAtomicOrderingAcquireOrStronger = ?; + + // isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering()) + // !isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering()) + bit IsAtomicOrderingReleaseOrStronger = ?; + + // cast<LoadSDNode>(N)->getMemoryVT() == MVT::<VT>; + // cast<StoreSDNode>(N)->getMemoryVT() == MVT::<VT>; + ValueType MemoryVT = ?; + // cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>; + // cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::<VT>; + ValueType ScalarMemoryVT = ?; } // OutPatFrag is a pattern fragment that is used as part of an output pattern @@ -676,12 +730,41 @@ class PatLeaf<dag frag, code pred = [{}], SDNodeXForm xform = NOOP_SDNodeXForm> // If FastIsel should ignore all instructions that have an operand of this type, // the FastIselShouldIgnore flag can be set. This is an optimization to reduce // the code size of the generated fast instruction selector. -class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm> - : PatFrag<(ops), (vt imm), [{}], xform> { +class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm, + SDNode ImmNode = imm> + : PatFrag<(ops), (vt ImmNode), [{}], xform> { let ImmediateCode = pred; bit FastIselShouldIgnore = 0; + + // Is the data type of the immediate an APInt? + bit IsAPInt = 0; + + // Is the data type of the immediate an APFloat? + bit IsAPFloat = 0; } +// An ImmLeaf except that Imm is an APInt. This is useful when you need to +// zero-extend the immediate instead of sign-extend it. +// +// Note that FastISel does not currently understand IntImmLeaf and will not +// generate code for rules that make use of it. As such, it does not make sense +// to replace ImmLeaf with IntImmLeaf. However, replacing PatLeaf with an +// IntImmLeaf will allow GlobalISel to import the rule. +class IntImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm> + : ImmLeaf<vt, pred, xform> { + let IsAPInt = 1; + let FastIselShouldIgnore = 1; +} + +// An ImmLeaf except that Imm is an APFloat. +// +// Note that FastISel does not currently understand FPImmLeaf and will not +// generate code for rules that make use of it. +class FPImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm> + : ImmLeaf<vt, pred, xform, fpimm> { + let IsAPFloat = 1; + let FastIselShouldIgnore = 1; +} // Leaf fragments. @@ -710,170 +793,215 @@ def ineg : PatFrag<(ops node:$in), (sub 0, node:$in)>; def null_frag : SDPatternOperator; // load fragments. -def unindexedload : PatFrag<(ops node:$ptr), (ld node:$ptr), [{ - return cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; -}]>; -def load : PatFrag<(ops node:$ptr), (unindexedload node:$ptr), [{ - return cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; -}]>; +def unindexedload : PatFrag<(ops node:$ptr), (ld node:$ptr)> { + let IsLoad = 1; + let IsUnindexed = 1; +} +def load : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> { + let IsLoad = 1; + let IsNonExtLoad = 1; +} // extending load fragments. -def extload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr), [{ - return cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; -}]>; -def sextload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr), [{ - return cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; -}]>; -def zextload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr), [{ - return cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; -}]>; +def extload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> { + let IsLoad = 1; + let IsAnyExtLoad = 1; +} +def sextload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> { + let IsLoad = 1; + let IsSignExtLoad = 1; +} +def zextload : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> { + let IsLoad = 1; + let IsZeroExtLoad = 1; +} -def extloadi1 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i1; -}]>; -def extloadi8 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; -def extloadi16 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; -def extloadi32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; -def extloadf32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::f32; -}]>; -def extloadf64 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::f64; -}]>; +def extloadi1 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i1; +} +def extloadi8 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i8; +} +def extloadi16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i16; +} +def extloadi32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i32; +} +def extloadf32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = f32; +} +def extloadf64 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = f64; +} -def sextloadi1 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i1; -}]>; -def sextloadi8 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; -def sextloadi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; -def sextloadi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; +def sextloadi1 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i1; +} +def sextloadi8 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i8; +} +def sextloadi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i16; +} +def sextloadi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i32; +} -def zextloadi1 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i1; -}]>; -def zextloadi8 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; -def zextloadi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; -def zextloadi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; +def zextloadi1 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i1; +} +def zextloadi8 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i8; +} +def zextloadi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i16; +} +def zextloadi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let MemoryVT = i32; +} -def extloadvi1 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i1; -}]>; -def extloadvi8 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; -def extloadvi16 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16; -}]>; -def extloadvi32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32; -}]>; -def extloadvf32 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::f32; -}]>; -def extloadvf64 : PatFrag<(ops node:$ptr), (extload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::f64; -}]>; +def extloadvi1 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i1; +} +def extloadvi8 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i8; +} +def extloadvi16 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i16; +} +def extloadvi32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i32; +} +def extloadvf32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = f32; +} +def extloadvf64 : PatFrag<(ops node:$ptr), (extload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = f64; +} -def sextloadvi1 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i1; -}]>; -def sextloadvi8 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; -def sextloadvi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16; -}]>; -def sextloadvi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32; -}]>; +def sextloadvi1 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i1; +} +def sextloadvi8 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i8; +} +def sextloadvi16 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i16; +} +def sextloadvi32 : PatFrag<(ops node:$ptr), (sextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i32; +} -def zextloadvi1 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i1; -}]>; -def zextloadvi8 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; -def zextloadvi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16; -}]>; -def zextloadvi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr), [{ - return cast<LoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32; -}]>; +def zextloadvi1 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i1; +} +def zextloadvi8 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i8; +} +def zextloadvi16 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i16; +} +def zextloadvi32 : PatFrag<(ops node:$ptr), (zextload node:$ptr)> { + let IsLoad = 1; + let ScalarMemoryVT = i32; +} // store fragments. def unindexedstore : PatFrag<(ops node:$val, node:$ptr), - (st node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; -}]>; + (st node:$val, node:$ptr)> { + let IsStore = 1; + let IsUnindexed = 1; +} def store : PatFrag<(ops node:$val, node:$ptr), - (unindexedstore node:$val, node:$ptr), [{ - return !cast<StoreSDNode>(N)->isTruncatingStore(); -}]>; + (unindexedstore node:$val, node:$ptr)> { + let IsStore = 1; + let IsTruncStore = 0; +} // truncstore fragments. def truncstore : PatFrag<(ops node:$val, node:$ptr), - (unindexedstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->isTruncatingStore(); -}]>; + (unindexedstore node:$val, node:$ptr)> { + let IsStore = 1; + let IsTruncStore = 1; +} def truncstorei8 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let MemoryVT = i8; +} def truncstorei16 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let MemoryVT = i16; +} def truncstorei32 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let MemoryVT = i32; +} def truncstoref32 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f32; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let MemoryVT = f32; +} def truncstoref64 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f64; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let MemoryVT = f64; +} def truncstorevi8 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let ScalarMemoryVT = i8; +} def truncstorevi16 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let ScalarMemoryVT = i16; +} def truncstorevi32 : PatFrag<(ops node:$val, node:$ptr), - (truncstore node:$val, node:$ptr), [{ - return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32; -}]>; + (truncstore node:$val, node:$ptr)> { + let IsStore = 1; + let ScalarMemoryVT = i32; +} // indexed store fragments. def istore : PatFrag<(ops node:$val, node:$base, node:$offset), - (ist node:$val, node:$base, node:$offset), [{ - return !cast<StoreSDNode>(N)->isTruncatingStore(); -}]>; + (ist node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let IsTruncStore = 0; +} def pre_store : PatFrag<(ops node:$val, node:$base, node:$offset), (istore node:$val, node:$base, node:$offset), [{ @@ -882,34 +1010,40 @@ def pre_store : PatFrag<(ops node:$val, node:$base, node:$offset), }]>; def itruncstore : PatFrag<(ops node:$val, node:$base, node:$offset), - (ist node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->isTruncatingStore(); -}]>; + (ist node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let IsTruncStore = 1; +} def pre_truncst : PatFrag<(ops node:$val, node:$base, node:$offset), (itruncstore node:$val, node:$base, node:$offset), [{ ISD::MemIndexedMode AM = cast<StoreSDNode>(N)->getAddressingMode(); return AM == ISD::PRE_INC || AM == ISD::PRE_DEC; }]>; def pre_truncsti1 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1; -}]>; + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i1; +} def pre_truncsti8 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i8; +} def pre_truncsti16 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i16; +} def pre_truncsti32 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i32; +} def pre_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f32; -}]>; + (pre_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = f32; +} def post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset), (istore node:$val, node:$ptr, node:$offset), [{ @@ -923,25 +1057,30 @@ def post_truncst : PatFrag<(ops node:$val, node:$base, node:$offset), return AM == ISD::POST_INC || AM == ISD::POST_DEC; }]>; def post_truncsti1 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1; -}]>; + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i1; +} def post_truncsti8 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i8; +} def post_truncsti16 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i16; +} def post_truncsti32 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = i32; +} def post_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncst node:$val, node:$base, node:$offset), [{ - return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f32; -}]>; + (post_truncst node:$val, node:$base, node:$offset)> { + let IsStore = 1; + let MemoryVT = f32; +} // nontemporal store fragments. def nontemporalstore : PatFrag<(ops node:$val, node:$ptr), @@ -1015,44 +1154,116 @@ def setle : PatFrag<(ops node:$lhs, node:$rhs), def setne : PatFrag<(ops node:$lhs, node:$rhs), (setcc node:$lhs, node:$rhs, SETNE)>; -def atomic_cmp_swap_8 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; -def atomic_cmp_swap_16 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; -def atomic_cmp_swap_32 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; -def atomic_cmp_swap_64 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64; -}]>; +multiclass binary_atomic_op_ord<SDNode atomic_op> { + def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingMonotonic = 1; + } + def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingAcquire = 1; + } + def #NAME#_release : PatFrag<(ops node:$ptr, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingRelease = 1; + } + def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingAcquireRelease = 1; + } + def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingSequentiallyConsistent = 1; + } +} + +multiclass ternary_atomic_op_ord<SDNode atomic_op> { + def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingMonotonic = 1; + } + def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingAcquire = 1; + } + def #NAME#_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingRelease = 1; + } + def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingAcquireRelease = 1; + } + def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let IsAtomicOrderingSequentiallyConsistent = 1; + } +} multiclass binary_atomic_op<SDNode atomic_op> { def _8 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8; - }]>; + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i8; + } def _16 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16; - }]>; + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i16; + } def _32 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32; - }]>; + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i32; + } def _64 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64; - }]>; + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i64; + } + + defm NAME#_8 : binary_atomic_op_ord<atomic_op>; + defm NAME#_16 : binary_atomic_op_ord<atomic_op>; + defm NAME#_32 : binary_atomic_op_ord<atomic_op>; + defm NAME#_64 : binary_atomic_op_ord<atomic_op>; +} + +multiclass ternary_atomic_op<SDNode atomic_op> { + def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i8; + } + def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i16; + } + def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i32; + } + def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = 1; + let MemoryVT = i64; + } + + defm NAME#_8 : ternary_atomic_op_ord<atomic_op>; + defm NAME#_16 : ternary_atomic_op_ord<atomic_op>; + defm NAME#_32 : ternary_atomic_op_ord<atomic_op>; + defm NAME#_64 : ternary_atomic_op_ord<atomic_op>; } defm atomic_load_add : binary_atomic_op<atomic_load_add>; @@ -1067,27 +1278,32 @@ defm atomic_load_max : binary_atomic_op<atomic_load_max>; defm atomic_load_umin : binary_atomic_op<atomic_load_umin>; defm atomic_load_umax : binary_atomic_op<atomic_load_umax>; defm atomic_store : binary_atomic_op<atomic_store>; +defm atomic_cmp_swap : ternary_atomic_op<atomic_cmp_swap>; def atomic_load_8 : PatFrag<(ops node:$ptr), - (atomic_load node:$ptr), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8; -}]>; + (atomic_load node:$ptr)> { + let IsAtomic = 1; + let MemoryVT = i8; +} def atomic_load_16 : PatFrag<(ops node:$ptr), - (atomic_load node:$ptr), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16; -}]>; + (atomic_load node:$ptr)> { + let IsAtomic = 1; + let MemoryVT = i16; +} def atomic_load_32 : PatFrag<(ops node:$ptr), - (atomic_load node:$ptr), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32; -}]>; + (atomic_load node:$ptr)> { + let IsAtomic = 1; + let MemoryVT = i32; +} def atomic_load_64 : PatFrag<(ops node:$ptr), - (atomic_load node:$ptr), [{ - return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64; -}]>; + (atomic_load node:$ptr)> { + let IsAtomic = 1; + let MemoryVT = i64; +} //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. diff --git a/contrib/llvm/include/llvm/Testing/Support/Error.h b/contrib/llvm/include/llvm/Testing/Support/Error.h index f23d289266ad..50889b9c66f5 100644 --- a/contrib/llvm/include/llvm/Testing/Support/Error.h +++ b/contrib/llvm/include/llvm/Testing/Support/Error.h @@ -22,17 +22,66 @@ namespace detail { ErrorHolder TakeError(Error Err); template <typename T> ExpectedHolder<T> TakeExpected(Expected<T> &Exp) { - llvm::detail::ExpectedHolder<T> Result; - auto &EH = static_cast<llvm::detail::ErrorHolder &>(Result); - EH = TakeError(Exp.takeError()); - if (Result.Success) - Result.Value = &(*Exp); - return Result; + return {TakeError(Exp.takeError()), Exp}; } template <typename T> ExpectedHolder<T> TakeExpected(Expected<T> &&Exp) { return TakeExpected(Exp); } + +template <typename T> +class ValueMatchesMono + : public testing::MatcherInterface<const ExpectedHolder<T> &> { +public: + explicit ValueMatchesMono(const testing::Matcher<T> &Matcher) + : Matcher(Matcher) {} + + bool MatchAndExplain(const ExpectedHolder<T> &Holder, + testing::MatchResultListener *listener) const override { + if (!Holder.Success) + return false; + + bool result = Matcher.MatchAndExplain(*Holder.Exp, listener); + + if (result) + return result; + *listener << "("; + Matcher.DescribeNegationTo(listener->stream()); + *listener << ")"; + return result; + } + + void DescribeTo(std::ostream *OS) const override { + *OS << "succeeded with value ("; + Matcher.DescribeTo(OS); + *OS << ")"; + } + + void DescribeNegationTo(std::ostream *OS) const override { + *OS << "did not succeed or value ("; + Matcher.DescribeNegationTo(OS); + *OS << ")"; + } + +private: + testing::Matcher<T> Matcher; +}; + +template<typename M> +class ValueMatchesPoly { +public: + explicit ValueMatchesPoly(const M &Matcher) : Matcher(Matcher) {} + + template <typename T> + operator testing::Matcher<const ExpectedHolder<T> &>() const { + return MakeMatcher( + new ValueMatchesMono<T>(testing::SafeMatcherCast<T>(Matcher))); + } + +private: + M Matcher; +}; + } // namespace detail #define EXPECT_THAT_ERROR(Err, Matcher) \ @@ -48,22 +97,11 @@ template <typename T> ExpectedHolder<T> TakeExpected(Expected<T> &&Exp) { MATCHER(Succeeded, "") { return arg.Success; } MATCHER(Failed, "") { return !arg.Success; } -MATCHER_P(HasValue, value, - "succeeded with value " + testing::PrintToString(value)) { - if (!arg.Success) { - *result_listener << "operation failed"; - return false; - } - - assert(arg.Value.hasValue()); - if (**arg.Value != value) { - *result_listener << "but \"" + testing::PrintToString(**arg.Value) + - "\" != " + testing::PrintToString(value); - return false; - } - - return true; +template <typename M> +detail::ValueMatchesPoly<M> HasValue(M Matcher) { + return detail::ValueMatchesPoly<M>(Matcher); } + } // namespace llvm #endif diff --git a/contrib/llvm/include/llvm/Testing/Support/SupportHelpers.h b/contrib/llvm/include/llvm/Testing/Support/SupportHelpers.h index c4dd414b80db..d7f0c7142b2c 100644 --- a/contrib/llvm/include/llvm/Testing/Support/SupportHelpers.h +++ b/contrib/llvm/include/llvm/Testing/Support/SupportHelpers.h @@ -22,7 +22,10 @@ struct ErrorHolder { }; template <typename T> struct ExpectedHolder : public ErrorHolder { - Optional<T *> Value; + ExpectedHolder(ErrorHolder Err, Expected<T> &Exp) + : ErrorHolder(std::move(Err)), Exp(Exp) {} + + Expected<T> &Exp; }; inline void PrintTo(const ErrorHolder &Err, std::ostream *Out) { @@ -35,8 +38,7 @@ inline void PrintTo(const ErrorHolder &Err, std::ostream *Out) { template <typename T> void PrintTo(const ExpectedHolder<T> &Item, std::ostream *Out) { if (Item.Success) { - *Out << "succeeded with value \"" << ::testing::PrintToString(**Item.Value) - << "\""; + *Out << "succeeded with value " << ::testing::PrintToString(*Item.Exp); } else { PrintTo(static_cast<const ErrorHolder &>(Item), Out); } diff --git a/contrib/llvm/include/llvm/Transforms/IPO.h b/contrib/llvm/include/llvm/Transforms/IPO.h index 39ceb19525b3..ce20a726b783 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO.h +++ b/contrib/llvm/include/llvm/Transforms/IPO.h @@ -216,6 +216,10 @@ ModulePass *createMetaRenamerPass(); /// manager. ModulePass *createBarrierNoopPass(); +/// createCalledValuePropagationPass - Attach metadata to indirct call sites +/// indicating the set of functions they may target at run-time. +ModulePass *createCalledValuePropagationPass(); + /// What to do with the summary when running passes that operate on it. enum class PassSummaryAction { None, ///< Do nothing. diff --git a/contrib/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h b/contrib/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h index 724ff72f3b5a..82ffc69a166e 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h @@ -12,6 +12,7 @@ #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/IR/PassManager.h" namespace llvm { @@ -26,6 +27,6 @@ public: LazyCallGraph &CG, CGSCCUpdateResult &UR); }; -} +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_IPO_ARGUMENTPROMOTION_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/CalledValuePropagation.h b/contrib/llvm/include/llvm/Transforms/IPO/CalledValuePropagation.h new file mode 100644 index 000000000000..352bdc7ac17f --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/IPO/CalledValuePropagation.h @@ -0,0 +1,35 @@ +//===- CalledValuePropagation.h - Propagate called values -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements a transformation that attaches !callees metadata to +// indirect call sites. For a given call site, the metadata, if present, +// indicates the set of functions the call site could possibly target at +// run-time. This metadata is added to indirect call sites when the set of +// possible targets can be determined by analysis and is known to be small. The +// analysis driving the transformation is similar to constant propagation and +// makes uses of the generic sparse propagation solver. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_CALLEDVALUEPROPAGATION_H +#define LLVM_TRANSFORMS_IPO_CALLEDVALUEPROPAGATION_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class CalledValuePropagationPass + : public PassInfoMixin<CalledValuePropagationPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &); +}; +} // namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_CALLEDVALUEPROPAGATION_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/ConstantMerge.h b/contrib/llvm/include/llvm/Transforms/IPO/ConstantMerge.h index 1d4da43f6a7b..e04d3ae1a40e 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/ConstantMerge.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/ConstantMerge.h @@ -20,16 +20,18 @@ #ifndef LLVM_TRANSFORMS_IPO_CONSTANTMERGE_H #define LLVM_TRANSFORMS_IPO_CONSTANTMERGE_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// A pass that merges duplicate global constants into a single constant. class ConstantMergePass : public PassInfoMixin<ConstantMergePass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_CONSTANTMERGE_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h b/contrib/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h index e179afa956f6..ba5666f20a9b 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/DeadArgumentElimination.h @@ -20,15 +20,21 @@ #ifndef LLVM_TRANSFORMS_IPO_DEADARGUMENTELIMINATION_H #define LLVM_TRANSFORMS_IPO_DEADARGUMENTELIMINATION_H -#include "llvm/IR/Module.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" - #include <map> #include <set> #include <string> +#include <tuple> namespace llvm { +class Module; +class Use; +class Value; + /// Eliminate dead arguments (and return values) from functions. class DeadArgumentEliminationPass : public PassInfoMixin<DeadArgumentEliminationPass> { @@ -37,12 +43,13 @@ public: /// argument. Used so that arguments and return values can be used /// interchangeably. struct RetOrArg { - RetOrArg(const Function *F, unsigned Idx, bool IsArg) - : F(F), Idx(Idx), IsArg(IsArg) {} const Function *F; unsigned Idx; bool IsArg; + RetOrArg(const Function *F, unsigned Idx, bool IsArg) + : F(F), Idx(Idx), IsArg(IsArg) {} + /// Make RetOrArg comparable, so we can put it into a map. bool operator<(const RetOrArg &O) const { return std::tie(F, Idx, IsArg) < std::tie(O.F, O.Idx, O.IsArg); @@ -67,16 +74,23 @@ public: /// thus become dead in the end. enum Liveness { Live, MaybeLive }; + DeadArgumentEliminationPass(bool ShouldHackArguments_ = false) + : ShouldHackArguments(ShouldHackArguments_) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &); + /// Convenience wrapper RetOrArg CreateRet(const Function *F, unsigned Idx) { return RetOrArg(F, Idx, false); } + /// Convenience wrapper RetOrArg CreateArg(const Function *F, unsigned Idx) { return RetOrArg(F, Idx, true); } - typedef std::multimap<RetOrArg, RetOrArg> UseMap; + using UseMap = std::multimap<RetOrArg, RetOrArg>; + /// This maps a return value or argument to any MaybeLive return values or /// arguments it uses. This allows the MaybeLive values to be marked live /// when any of its users is marked live. @@ -93,25 +107,21 @@ public: /// directly to F. UseMap Uses; - typedef std::set<RetOrArg> LiveSet; - typedef std::set<const Function *> LiveFuncSet; + using LiveSet = std::set<RetOrArg>; + using LiveFuncSet = std::set<const Function *>; /// This set contains all values that have been determined to be live. LiveSet LiveValues; + /// This set contains all values that are cannot be changed in any way. LiveFuncSet LiveFunctions; - typedef SmallVector<RetOrArg, 5> UseVector; + using UseVector = SmallVector<RetOrArg, 5>; /// This allows this pass to do double-duty as the dead arg hacking pass /// (used only by bugpoint). bool ShouldHackArguments = false; -public: - DeadArgumentEliminationPass(bool ShouldHackArguments_ = false) - : ShouldHackArguments(ShouldHackArguments_) {} - PreservedAnalyses run(Module &M, ModuleAnalysisManager &); - private: Liveness MarkIfNotLive(RetOrArg Use, UseVector &MaybeLiveUses); Liveness SurveyUse(const Use *U, UseVector &MaybeLiveUses, @@ -128,6 +138,7 @@ private: bool DeleteDeadVarargs(Function &Fn); bool RemoveDeadArgumentsFromCallers(Function &Fn); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_DEADARGUMENTELIMINATION_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/ElimAvailExtern.h b/contrib/llvm/include/llvm/Transforms/IPO/ElimAvailExtern.h index 88a0e9bd8ce0..94cb954fd2d5 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/ElimAvailExtern.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/ElimAvailExtern.h @@ -15,17 +15,19 @@ #ifndef LLVM_TRANSFORMS_IPO_ELIMAVAILEXTERN_H #define LLVM_TRANSFORMS_IPO_ELIMAVAILEXTERN_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// A pass that transforms external global definitions into declarations. class EliminateAvailableExternallyPass : public PassInfoMixin<EliminateAvailableExternallyPass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_ELIMAVAILEXTERN_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h b/contrib/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h index 36dd06b85b41..dc9f18c79410 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/FunctionAttrs.h @@ -1,4 +1,4 @@ -//===-- FunctionAttrs.h - Compute function attrs --------------------------===// +//===- FunctionAttrs.h - Compute function attributes ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,9 +6,11 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// Provides passes for computing function attributes based on interprocedural /// analyses. +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H @@ -21,6 +23,9 @@ namespace llvm { class AAResults; +class Function; +class Module; +class Pass; /// The three kinds of memory access relevant to 'readonly' and /// 'readnone' attributes. @@ -66,6 +71,7 @@ class ReversePostOrderFunctionAttrsPass public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/contrib/llvm/include/llvm/Transforms/IPO/FunctionImport.h index de35cdf052e1..39e5b5c8ae6f 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/FunctionImport.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/FunctionImport.h @@ -7,23 +7,26 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_FUNCTIONIMPORT_H -#define LLVM_FUNCTIONIMPORT_H +#ifndef LLVM_TRANSFORMS_IPO_FUNCTIONIMPORT_H +#define LLVM_TRANSFORMS_IPO_FUNCTIONIMPORT_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Error.h" - #include <functional> #include <map> +#include <memory> +#include <string> +#include <system_error> #include <unordered_set> #include <utility> namespace llvm { -class LLVMContext; -class GlobalValueSummary; + class Module; /// The function importer is automatically importing function from other modules @@ -34,19 +37,19 @@ public: /// containing all the functions to import for a source module. /// The keys is the GUID identifying a function to import, and the value /// is the threshold applied when deciding to import it. - typedef std::map<GlobalValue::GUID, unsigned> FunctionsToImportTy; + using FunctionsToImportTy = std::map<GlobalValue::GUID, unsigned>; /// The map contains an entry for every module to import from, the key being /// the module identifier to pass to the ModuleLoader. The value is the set of /// functions to import. - typedef StringMap<FunctionsToImportTy> ImportMapTy; + using ImportMapTy = StringMap<FunctionsToImportTy>; /// The set contains an entry for every global value the module exports. - typedef std::unordered_set<GlobalValue::GUID> ExportSetTy; + using ExportSetTy = std::unordered_set<GlobalValue::GUID>; /// A function of this type is used to load modules referenced by the index. - typedef std::function<Expected<std::unique_ptr<Module>>(StringRef Identifier)> - ModuleLoaderTy; + using ModuleLoaderTy = + std::function<Expected<std::unique_ptr<Module>>(StringRef Identifier)>; /// Create a Function Importer. FunctionImporter(const ModuleSummaryIndex &Index, ModuleLoaderTy ModuleLoader) @@ -95,6 +98,15 @@ void ComputeCrossModuleImportForModule( StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList); +/// Mark all external summaries in \p Index for import into the given module. +/// Used for distributed builds using a distributed index. +/// +/// \p ImportList will be populated with a map that can be passed to +/// FunctionImporter::importFunctions() above (see description there). +void ComputeCrossModuleImportForModuleFromIndex( + StringRef ModulePath, const ModuleSummaryIndex &Index, + FunctionImporter::ImportMapTy &ImportList); + /// Compute all the symbols that are "dead": i.e these that can't be reached /// in the graph from any of the given symbols listed in /// \p GUIDPreservedSymbols. @@ -132,6 +144,7 @@ void thinLTOResolveWeakForLinkerModule(Module &TheModule, /// during global summary-based analysis. void thinLTOInternalizeModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals); -} -#endif // LLVM_FUNCTIONIMPORT_H +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_FUNCTIONIMPORT_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/GlobalDCE.h b/contrib/llvm/include/llvm/Transforms/IPO/GlobalDCE.h index 9ca939c15b62..7ca241f4645a 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/GlobalDCE.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/GlobalDCE.h @@ -35,7 +35,7 @@ private: SmallPtrSet<GlobalValue*, 32> AliveGlobals; /// Global -> Global that uses this global. - std::unordered_multimap<GlobalValue *, GlobalValue *> GVDependencies; + DenseMap<GlobalValue *, SmallPtrSet<GlobalValue *, 4>> GVDependencies; /// Constant -> Globals that use this global cache. std::unordered_map<Constant *, SmallPtrSet<GlobalValue *, 8>> diff --git a/contrib/llvm/include/llvm/Transforms/IPO/GlobalOpt.h b/contrib/llvm/include/llvm/Transforms/IPO/GlobalOpt.h index ab9116810be1..5b4878604eab 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/GlobalOpt.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/GlobalOpt.h @@ -16,17 +16,18 @@ #ifndef LLVM_TRANSFORMS_IPO_GLOBALOPT_H #define LLVM_TRANSFORMS_IPO_GLOBALOPT_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// Optimize globals that never have their address taken. class GlobalOptPass : public PassInfoMixin<GlobalOptPass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} +} // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_GLOBALOPT_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/GlobalSplit.h b/contrib/llvm/include/llvm/Transforms/IPO/GlobalSplit.h index fb2c2d27338e..56cefb7886fe 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/GlobalSplit.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/GlobalSplit.h @@ -17,14 +17,18 @@ #ifndef LLVM_TRANSFORMS_IPO_GLOBALSPLIT_H #define LLVM_TRANSFORMS_IPO_GLOBALSPLIT_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { + +class Module; + /// Pass to perform split of global variables. class GlobalSplitPass : public PassInfoMixin<GlobalSplitPass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} + +} // end namespace llvm + #endif // LLVM_TRANSFORMS_IPO_GLOBALSPLIT_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/Inliner.h b/contrib/llvm/include/llvm/Transforms/IPO/Inliner.h index b3ca5156e388..eda8cf462b50 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -14,15 +14,15 @@ #include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/LazyCallGraph.h" -#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/PassManager.h" #include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h" +#include <utility> namespace llvm { + class AssumptionCacheTracker; -class CallSite; -class DataLayout; -class InlineCost; -class OptimizationRemarkEmitter; +class CallGraph; class ProfileSummaryInfo; /// This class contains all of the helper code which is used to perform the @@ -44,6 +44,7 @@ struct LegacyInlinerBase : public CallGraphSCCPass { bool runOnSCC(CallGraphSCC &SCC) override; using llvm::Pass::doFinalization; + /// Remove now-dead linkonce functions at the end of processing to avoid /// breaking the SCC traversal. bool doFinalization(CallGraph &CG) override; @@ -69,7 +70,7 @@ struct LegacyInlinerBase : public CallGraphSCCPass { private: // Insert @llvm.lifetime intrinsics. - bool InsertLifetime; + bool InsertLifetime = true; protected: AssumptionCacheTracker *ACT; @@ -103,6 +104,6 @@ private: InlineParams Params; }; -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_IPO_INLINER_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h b/contrib/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h index a2b888ce9ffa..3bcfe65df550 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h @@ -16,7 +16,6 @@ #define LLVM_TRANSFORMS_IPO_LOWERTYPETESTS_H #include "llvm/ADT/SmallVector.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include <cstdint> #include <cstring> @@ -26,6 +25,7 @@ namespace llvm { +class Module; class raw_ostream; namespace lowertypetests { diff --git a/contrib/llvm/include/llvm/Transforms/IPO/PartialInlining.h b/contrib/llvm/include/llvm/Transforms/IPO/PartialInlining.h index 15407fc36a22..ec6dd36dae06 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/PartialInlining.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/PartialInlining.h @@ -1,4 +1,4 @@ -//===- PartialInlining.h - Inline parts of functions --------------------===// +//===- PartialInlining.h - Inline parts of functions ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,15 +15,18 @@ #ifndef LLVM_TRANSFORMS_IPO_PARTIALINLINING_H #define LLVM_TRANSFORMS_IPO_PARTIALINLINING_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Module; + /// Pass to remove unused function declarations. class PartialInlinerPass : public PassInfoMixin<PartialInlinerPass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &); }; -} + +} // end namespace llvm + #endif // LLVM_TRANSFORMS_IPO_PARTIALINLINING_H diff --git a/contrib/llvm/include/llvm/Transforms/IPO/SCCP.h b/contrib/llvm/include/llvm/Transforms/IPO/SCCP.h index 7082006f14a6..fdb7865fbac3 100644 --- a/contrib/llvm/include/llvm/Transforms/IPO/SCCP.h +++ b/contrib/llvm/include/llvm/Transforms/IPO/SCCP.h @@ -21,14 +21,18 @@ #ifndef LLVM_TRANSFORMS_IPO_SCCP_H #define LLVM_TRANSFORMS_IPO_SCCP_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" namespace llvm { + +class Module; + /// Pass to perform interprocedural constant propagation. class IPSCCPPass : public PassInfoMixin<IPSCCPPass> { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; -} + +} // end namespace llvm + #endif // LLVM_TRANSFORMS_IPO_SCCP_H diff --git a/contrib/llvm/include/llvm/Transforms/Instrumentation.h b/contrib/llvm/include/llvm/Transforms/Instrumentation.h index f2fc6dc8dad5..cd6b770f76ac 100644 --- a/contrib/llvm/include/llvm/Transforms/Instrumentation.h +++ b/contrib/llvm/include/llvm/Transforms/Instrumentation.h @@ -22,24 +22,11 @@ #include <string> #include <vector> -#if defined(__GNUC__) && defined(__linux__) && !defined(ANDROID) -inline void *getDFSanArgTLSPtrForJIT() { - extern __thread __attribute__((tls_model("initial-exec"))) - void *__dfsan_arg_tls; - return (void *)&__dfsan_arg_tls; -} - -inline void *getDFSanRetValTLSPtrForJIT() { - extern __thread __attribute__((tls_model("initial-exec"))) - void *__dfsan_retval_tls; - return (void *)&__dfsan_retval_tls; -} -#endif - namespace llvm { class FunctionPass; class ModulePass; +class OptimizationRemarkEmitter; /// Instrumentation passes often insert conditional checks into entry blocks. /// Call this function before splitting the entry block to move instructions @@ -90,9 +77,12 @@ ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false); FunctionPass *createPGOMemOPSizeOptLegacyPass(); -// Helper function to check if it is legal to promote indirect call \p Inst -// to a direct call of function \p F. Stores the reason in \p Reason. -bool isLegalToPromote(Instruction *Inst, Function *F, const char **Reason); +// The pgo-specific indirect call promotion function declared below is used by +// the pgo-driven indirect call promotion and sample profile passes. It's a +// wrapper around llvm::promoteCall, et al. that additionally computes !prof +// metadata. We place it in a pgo namespace so it's not confused with the +// generic utilities. +namespace pgo { // Helper function that transforms Inst (either an indirect-call instruction, or // an invoke instruction , to a conditional call to F. This is like: @@ -109,7 +99,9 @@ bool isLegalToPromote(Instruction *Inst, Function *F, const char **Reason); // Returns the promoted direct call instruction. Instruction *promoteIndirectCall(Instruction *Inst, Function *F, uint64_t Count, uint64_t TotalCount, - bool AttachProfToDirectCall); + bool AttachProfToDirectCall, + OptimizationRemarkEmitter *ORE); +} // namespace pgo /// Options for the frontend instrumentation based profiling pass. struct InstrProfOptions { @@ -141,6 +133,8 @@ ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false, FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0, bool Recover = false); +FunctionPass *createHWAddressSanitizerPass(); + // Insert ThreadSanitizer (race detection) instrumentation FunctionPass *createThreadSanitizerPass(); @@ -181,7 +175,9 @@ struct SanitizerCoverageOptions { bool TracePC = false; bool TracePCGuard = false; bool Inline8bitCounters = false; + bool PCTable = false; bool NoPrune = false; + bool StackDepth = false; SanitizerCoverageOptions() = default; }; @@ -190,18 +186,6 @@ struct SanitizerCoverageOptions { ModulePass *createSanitizerCoverageModulePass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); -#if defined(__GNUC__) && defined(__linux__) && !defined(ANDROID) -inline ModulePass *createDataFlowSanitizerPassForJIT( - const std::vector<std::string> &ABIListFiles = std::vector<std::string>()) { - return createDataFlowSanitizerPass(ABIListFiles, getDFSanArgTLSPtrForJIT, - getDFSanRetValTLSPtrForJIT); -} -#endif - -// BoundsChecking - This pass instruments the code to perform run-time bounds -// checking on loads, stores, and other memory intrinsics. -FunctionPass *createBoundsCheckingPass(); - /// \brief Calculate what to divide by to scale counts. /// /// Given the maximum count, calculate a divisor that will scale all the diff --git a/contrib/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/contrib/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h new file mode 100644 index 000000000000..3d4f62c121c2 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h @@ -0,0 +1,29 @@ +//===- BoundsChecking.h - Bounds checking instrumentation -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_BOUNDSCHECKING_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_BOUNDSCHECKING_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// A pass to instrument code and perform run-time bounds checking on loads, +/// stores, and other memory intrinsics. +struct BoundsCheckingPass : PassInfoMixin<BoundsCheckingPass> { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + + +/// Legacy pass creation function for the above pass. +FunctionPass *createBoundsCheckingLegacyPass(); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_INSTRUMENTATION_BOUNDSCHECKING_H diff --git a/contrib/llvm/include/llvm/Transforms/PGOInstrumentation.h b/contrib/llvm/include/llvm/Transforms/PGOInstrumentation.h index 19263f0f8071..c2cc76c422da 100644 --- a/contrib/llvm/include/llvm/Transforms/PGOInstrumentation.h +++ b/contrib/llvm/include/llvm/Transforms/PGOInstrumentation.h @@ -1,4 +1,4 @@ -//===- Transforms/PGOInstrumentation.h - PGO gen/use passes ---*- C++ -*-===// +//===- Transforms/PGOInstrumentation.h - PGO gen/use passes -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,19 +6,27 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides the interface for IR based instrumentation passes ( /// (profile-gen, and profile-use). +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_PGOINSTRUMENTATION_H #define LLVM_TRANSFORMS_PGOINSTRUMENTATION_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Instrumentation.h" +#include <cstdint> +#include <string> namespace llvm { +class Function; +class Instruction; +class Module; + /// The instrumentation (profile-instr-gen) pass for IR based PGO. class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> { public: @@ -28,9 +36,10 @@ public: /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin<PGOInstrumentationUse> { public: - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); PGOInstrumentationUse(std::string Filename = ""); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + private: std::string ProfileFileName; }; @@ -40,6 +49,7 @@ class PGOIndirectCallPromotion : public PassInfoMixin<PGOIndirectCallPromotion> public: PGOIndirectCallPromotion(bool IsInLTO = false, bool SamplePGO = false) : InLTO(IsInLTO), SamplePGO(SamplePGO) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: @@ -50,12 +60,16 @@ private: /// The profile size based optimization pass for memory intrinsics. class PGOMemOPSizeOpt : public PassInfoMixin<PGOMemOPSizeOpt> { public: - PGOMemOPSizeOpt() {} + PGOMemOPSizeOpt() = default; + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts, uint64_t MaxCount); -} // End llvm namespace -#endif +void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_PGOINSTRUMENTATION_H diff --git a/contrib/llvm/include/llvm/Transforms/SampleProfile.h b/contrib/llvm/include/llvm/Transforms/SampleProfile.h index c984fe74ba93..f5a8590e14a1 100644 --- a/contrib/llvm/include/llvm/Transforms/SampleProfile.h +++ b/contrib/llvm/include/llvm/Transforms/SampleProfile.h @@ -1,4 +1,4 @@ -//===- Transforms/SampleProfile.h - SamplePGO pass--------------*- C++ -*-===// +//===- Transforms/SampleProfile.h - SamplePGO pass --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,26 +6,35 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides the interface for the sampled PGO loader pass. +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SAMPLEPROFILE_H #define LLVM_TRANSFORMS_SAMPLEPROFILE_H #include "llvm/IR/PassManager.h" +#include <string> namespace llvm { +class Module; + /// The sample profiler data loader pass. class SampleProfileLoaderPass : public PassInfoMixin<SampleProfileLoaderPass> { public: + SampleProfileLoaderPass(std::string File = "", bool IsThinLTOPreLink = false) + : ProfileFileName(File), IsThinLTOPreLink(IsThinLTOPreLink) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - SampleProfileLoaderPass(std::string File = "") : ProfileFileName(File) {} private: std::string ProfileFileName; + bool IsThinLTOPreLink; }; -} // End llvm namespace -#endif +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SAMPLEPROFILE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar.h b/contrib/llvm/include/llvm/Transforms/Scalar.h index 1913a9d5da02..49186bc5cd66 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar.h @@ -73,6 +73,14 @@ FunctionPass *createDeadCodeEliminationPass(); // FunctionPass *createDeadStoreEliminationPass(); + +//===----------------------------------------------------------------------===// +// +// CallSiteSplitting - This pass split call-site based on its known argument +// values. +FunctionPass *createCallSiteSplittingPass(); + + //===----------------------------------------------------------------------===// // // AggressiveDCE - This pass uses the SSA based Aggressive DCE algorithm. This @@ -184,7 +192,7 @@ Pass *createLoopInstSimplifyPass(); // Pass *createLoopUnrollPass(int OptLevel = 2, int Threshold = -1, int Count = -1, int AllowPartial = -1, int Runtime = -1, - int UpperBound = -1); + int UpperBound = -1, int AllowPeeling = -1); // Create an unrolling pass for full unrolling that uses exact trip count only. Pass *createSimpleLoopUnrollPass(int OptLevel = 2); @@ -255,18 +263,12 @@ FunctionPass *createJumpThreadingPass(int Threshold = -1); //===----------------------------------------------------------------------===// // // CFGSimplification - Merge basic blocks, eliminate unreachable blocks, -// simplify terminator instructions, etc... +// simplify terminator instructions, convert switches to lookup tables, etc. // FunctionPass *createCFGSimplificationPass( - int Threshold = -1, std::function<bool(const Function &)> Ftor = nullptr); - -//===----------------------------------------------------------------------===// -// -// LateCFGSimplification - Like CFGSimplification, but may also -// convert switches to lookup tables. -// -FunctionPass *createLateCFGSimplificationPass( - int Threshold = -1, std::function<bool(const Function &)> Ftor = nullptr); + unsigned Threshold = 1, bool ForwardSwitchCond = false, + bool ConvertSwitch = false, bool KeepLoops = true, bool SinkCommon = false, + std::function<bool(const Function &)> Ftor = nullptr); //===----------------------------------------------------------------------===// // @@ -377,6 +379,12 @@ FunctionPass *createNewGVNPass(); //===----------------------------------------------------------------------===// // +// DivRemPairs - Hoist/decompose integer division and remainder instructions. +// +FunctionPass *createDivRemPairsPass(); + +//===----------------------------------------------------------------------===// +// // MemCpyOpt - This pass performs optimizations related to eliminating memcpy // calls and/or combining multiple stores into memset's. // @@ -422,6 +430,12 @@ Pass *createLowerGuardIntrinsicPass(); //===----------------------------------------------------------------------===// // +// MergeICmps - Merge integer comparison chains into a memcmp +// +Pass *createMergeICmpsPass(); + +//===----------------------------------------------------------------------===// +// // ValuePropagation - Propagate CFG-derived value information // Pass *createCorrelatedValuePropagationPass(); @@ -507,7 +521,7 @@ FunctionPass *createPlaceSafepointsPass(); // RewriteStatepointsForGC - Rewrite any gc.statepoints which do not yet have // explicit relocations to include explicit relocations. // -ModulePass *createRewriteStatepointsForGCPass(); +ModulePass *createRewriteStatepointsForGCLegacyPass(); //===----------------------------------------------------------------------===// // @@ -568,6 +582,16 @@ ModulePass *createNameAnonGlobalPass(); // used. // FunctionPass *createLibCallsShrinkWrapPass(); + +//===----------------------------------------------------------------------===// +// +// EntryExitInstrumenter pass - Instrument function entry/exit with calls to +// mcount(), @__cyg_profile_func_{enter,exit} and the like. There are two +// variants, intended to run pre- and post-inlining, respectively. +// +FunctionPass *createEntryExitInstrumenterPass(); +FunctionPass *createPostInlineEntryExitInstrumenterPass(); + } // End llvm namespace #endif diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/ADCE.h b/contrib/llvm/include/llvm/Transforms/Scalar/ADCE.h index b9b7e1c0c99f..f98af62c1a76 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/ADCE.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/ADCE.h @@ -1,4 +1,4 @@ -//===- ADCE.h - Aggressive dead code elimination --------------------------===// +//===- ADCE.h - Aggressive dead code elimination ----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -17,11 +17,12 @@ #ifndef LLVM_TRANSFORMS_SCALAR_ADCE_H #define LLVM_TRANSFORMS_SCALAR_ADCE_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// A DCE pass that assumes instructions are dead until proven otherwise. /// /// This pass eliminates dead code by optimistically assuming that all @@ -31,6 +32,7 @@ namespace llvm { struct ADCEPass : PassInfoMixin<ADCEPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_ADCE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/contrib/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h new file mode 100644 index 000000000000..5ab951a49f2c --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h @@ -0,0 +1,29 @@ +//===- CallSiteSplitting..h - Callsite Splitting ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H +#define LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H + +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" +#include <vector> + +namespace llvm { + +struct CallSiteSplittingPass : PassInfoMixin<CallSiteSplittingPass> { + /// \brief Run the pass over the function. + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h b/contrib/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h index a2a9afc083a0..d3322dc1c414 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h @@ -1,4 +1,4 @@ -//===-- ConstantHoisting.h - Prepare code for expensive constants ---------===// +//==- ConstantHoisting.h - Prepare code for expensive constants --*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -31,41 +31,53 @@ // This optimization is only applied to integer constants in instructions and // simple (this means not nested) constant cast expressions. For example: // %0 = load i64* inttoptr (i64 big_constant to i64*) +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_CONSTANTHOISTING_H #define LLVM_TRANSFORMS_SCALAR_CONSTANTHOISTING_H -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/Dominators.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" +#include <algorithm> +#include <vector> namespace llvm { +class BasicBlock; +class BlockFrequencyInfo; +class Constant; +class ConstantInt; +class DominatorTree; +class Function; +class Instruction; +class TargetTransformInfo; + /// A private "module" namespace for types and utilities used by /// ConstantHoisting. These are implementation details and should not be used by /// clients. namespace consthoist { + /// \brief Keeps track of the user of a constant and the operand index where the /// constant is used. struct ConstantUser { Instruction *Inst; unsigned OpndIdx; - ConstantUser(Instruction *Inst, unsigned Idx) : Inst(Inst), OpndIdx(Idx) { } + ConstantUser(Instruction *Inst, unsigned Idx) : Inst(Inst), OpndIdx(Idx) {} }; -typedef SmallVector<ConstantUser, 8> ConstantUseListType; +using ConstantUseListType = SmallVector<ConstantUser, 8>; /// \brief Keeps track of a constant candidate and its uses. struct ConstantCandidate { ConstantUseListType Uses; ConstantInt *ConstInt; - unsigned CumulativeCost; + unsigned CumulativeCost = 0; - ConstantCandidate(ConstantInt *ConstInt) - : ConstInt(ConstInt), CumulativeCost(0) { } + ConstantCandidate(ConstantInt *ConstInt) : ConstInt(ConstInt) {} /// \brief Add the user to the use list and update the cost. void addUser(Instruction *Inst, unsigned Idx, unsigned Cost) { @@ -81,17 +93,18 @@ struct RebasedConstantInfo { Constant *Offset; RebasedConstantInfo(ConstantUseListType &&Uses, Constant *Offset) - : Uses(std::move(Uses)), Offset(Offset) { } + : Uses(std::move(Uses)), Offset(Offset) {} }; -typedef SmallVector<RebasedConstantInfo, 4> RebasedConstantListType; +using RebasedConstantListType = SmallVector<RebasedConstantInfo, 4>; /// \brief A base constant and all its rebased constants. struct ConstantInfo { ConstantInt *BaseConstant; RebasedConstantListType RebasedConstants; }; -} + +} // end namespace consthoist class ConstantHoistingPass : public PassInfoMixin<ConstantHoistingPass> { public: @@ -108,8 +121,8 @@ public: } private: - typedef DenseMap<ConstantInt *, unsigned> ConstCandMapType; - typedef std::vector<consthoist::ConstantCandidate> ConstCandVecType; + using ConstCandMapType = DenseMap<ConstantInt *, unsigned>; + using ConstCandVecType = std::vector<consthoist::ConstantCandidate>; const TargetTransformInfo *TTI; DominatorTree *DT; @@ -148,6 +161,7 @@ private: void deleteDeadCastInst() const; bool optimizeConstants(Function &Fn); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_CONSTANTHOISTING_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h b/contrib/llvm/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h index 38816bbed068..20930699b557 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h @@ -1,4 +1,4 @@ -//===---- CorrelatedValuePropagation.h --------------------------*- C++ -*-===// +//===- CorrelatedValuePropagation.h -----------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -10,15 +10,17 @@ #ifndef LLVM_TRANSFORMS_SCALAR_CORRELATEDVALUEPROPAGATION_H #define LLVM_TRANSFORMS_SCALAR_CORRELATEDVALUEPROPAGATION_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + struct CorrelatedValuePropagationPass : PassInfoMixin<CorrelatedValuePropagationPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_CORRELATEDVALUEPROPAGATION_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/DeadStoreElimination.h b/contrib/llvm/include/llvm/Transforms/Scalar/DeadStoreElimination.h index 3ae999dfb542..cfeb21814232 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/DeadStoreElimination.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/DeadStoreElimination.h @@ -1,4 +1,4 @@ -//===- DeadStoreElimination.h - Fast Dead Store Elimination -------------===// +//===- DeadStoreElimination.h - Fast Dead Store Elimination -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,20 +15,22 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_TRANSFORMS_SCALAR_DSE_H -#define LLVM_TRANSFORMS_SCALAR_DSE_H +#ifndef LLVM_TRANSFORMS_SCALAR_DEADSTOREELIMINATION_H +#define LLVM_TRANSFORMS_SCALAR_DEADSTOREELIMINATION_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// This class implements a trivial dead store elimination. We consider /// only the redundant stores that are local to a single Basic Block. class DSEPass : public PassInfoMixin<DSEPass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); }; -} -#endif // LLVM_TRANSFORMS_SCALAR_DSE_H +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_DEADSTOREELIMINATION_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/DivRemPairs.h b/contrib/llvm/include/llvm/Transforms/Scalar/DivRemPairs.h new file mode 100644 index 000000000000..0a4346f33b12 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Scalar/DivRemPairs.h @@ -0,0 +1,31 @@ +//===- DivRemPairs.h - Hoist/decompose integer division and remainder -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass hoists and/or decomposes integer division and remainder +// instructions to enable CFG improvements and better codegen. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_DIVREMPAIRS_H +#define LLVM_TRANSFORMS_SCALAR_DIVREMPAIRS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Hoist/decompose integer division and remainder instructions to enable CFG +/// improvements and better codegen. +struct DivRemPairsPass : public PassInfoMixin<DivRemPairsPass> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &); +}; + +} +#endif // LLVM_TRANSFORMS_SCALAR_DIVREMPAIRS_H + diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h b/contrib/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h index 969ab78bfd19..dca3b2dbf04f 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/EarlyCSE.h @@ -6,19 +6,21 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides the interface for a simple, fast CSE pass. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_EARLYCSE_H #define LLVM_TRANSFORMS_SCALAR_EARLYCSE_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// \brief A simple and fast domtree-based CSE pass. /// /// This pass does a simple depth-first walk over the dominator tree, @@ -35,6 +37,6 @@ struct EarlyCSEPass : PassInfoMixin<EarlyCSEPass> { bool UseMemorySSA; }; -} +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_SCALAR_EARLYCSE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h b/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h index f25ab40640df..440d3f67c35a 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -18,26 +18,48 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Transforms/Utils/OrderedInstructions.h" +#include <cstdint> +#include <utility> +#include <vector> namespace llvm { + +class AssumptionCache; +class BasicBlock; +class BranchInst; +class CallInst; +class Constant; +class ExtractValueInst; +class Function; +class FunctionPass; +class IntrinsicInst; +class LoadInst; +class LoopInfo; class OptimizationRemarkEmitter; +class PHINode; +class TargetLibraryInfo; +class Value; /// A private "module" namespace for types and utilities used by GVN. These /// are implementation details and should not be used by clients. namespace gvn LLVM_LIBRARY_VISIBILITY { + struct AvailableValue; struct AvailableValueInBlock; class GVNLegacyPass; -} + +} // end namespace gvn /// The core GVN pass object. /// @@ -45,6 +67,7 @@ class GVNLegacyPass; /// this particular pass here. class GVN : public PassInfoMixin<GVN> { public: + struct Expression; /// \brief Run the pass over the function. PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -60,25 +83,45 @@ public: AliasAnalysis *getAliasAnalysis() const { return VN.getAliasAnalysis(); } MemoryDependenceResults &getMemDep() const { return *MD; } - struct Expression; - /// This class holds the mapping between values and value numbers. It is used /// as an efficient mechanism to determine the expression-wise equivalence of /// two values. class ValueTable { DenseMap<Value *, uint32_t> valueNumbering; DenseMap<Expression, uint32_t> expressionNumbering; + + // Expressions is the vector of Expression. ExprIdx is the mapping from + // value number to the index of Expression in Expressions. We use it + // instead of a DenseMap because filling such mapping is faster than + // filling a DenseMap and the compile time is a little better. + uint32_t nextExprNumber; + + std::vector<Expression> Expressions; + std::vector<uint32_t> ExprIdx; + + // Value number to PHINode mapping. Used for phi-translate in scalarpre. + DenseMap<uint32_t, PHINode *> NumberingPhi; + + // Cache for phi-translate in scalarpre. + using PhiTranslateMap = + DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>; + PhiTranslateMap PhiTranslateTable; + AliasAnalysis *AA; MemoryDependenceResults *MD; DominatorTree *DT; - uint32_t nextValueNumber; + uint32_t nextValueNumber = 1; Expression createExpr(Instruction *I); Expression createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate, Value *LHS, Value *RHS); Expression createExtractvalueExpr(ExtractValueInst *EI); uint32_t lookupOrAddCall(CallInst *C); + uint32_t phiTranslateImpl(const BasicBlock *BB, const BasicBlock *PhiBlock, + uint32_t Num, GVN &Gvn); + std::pair<uint32_t, bool> assignExpNewValueNum(Expression &exp); + bool areAllValsInBB(uint32_t num, const BasicBlock *BB, GVN &Gvn); public: ValueTable(); @@ -87,9 +130,12 @@ public: ~ValueTable(); uint32_t lookupOrAdd(Value *V); - uint32_t lookup(Value *V) const; + uint32_t lookup(Value *V, bool Verify = true) const; uint32_t lookupOrAddCmp(unsigned Opcode, CmpInst::Predicate Pred, Value *LHS, Value *RHS); + uint32_t phiTranslate(const BasicBlock *BB, const BasicBlock *PhiBlock, + uint32_t Num, GVN &Gvn); + void eraseTranslateCacheEntry(uint32_t Num, const BasicBlock &CurrBlock); bool exists(Value *V) const; void add(Value *V, uint32_t num); void clear(); @@ -112,7 +158,11 @@ private: AssumptionCache *AC; SetVector<BasicBlock *> DeadBlocks; OptimizationRemarkEmitter *ORE; + // Maps a block to the topmost instruction with implicit control flow in it. + DenseMap<const BasicBlock *, const Instruction *> + FirstImplicitControlFlowInsts; + OrderedInstructions *OI; ValueTable VN; /// A mapping from value numbers to lists of Value*'s that @@ -128,12 +178,16 @@ private: // Block-local map of equivalent values to their leader, does not // propagate to any successors. Entries added mid-block are applied // to the remaining instructions in the block. - SmallMapVector<llvm::Value *, llvm::Constant *, 4> ReplaceWithConstMap; + SmallMapVector<Value *, Constant *, 4> ReplaceWithConstMap; SmallVector<Instruction *, 8> InstrsToErase; - typedef SmallVector<NonLocalDepResult, 64> LoadDepVect; - typedef SmallVector<gvn::AvailableValueInBlock, 64> AvailValInBlkVect; - typedef SmallVector<BasicBlock *, 64> UnavailBlkVect; + // Map the block to reversed postorder traversal number. It is used to + // find back edge easily. + DenseMap<const BasicBlock *, uint32_t> BlockRPONumber; + + using LoadDepVect = SmallVector<NonLocalDepResult, 64>; + using AvailValInBlkVect = SmallVector<gvn::AvailableValueInBlock, 64>; + using UnavailBlkVect = SmallVector<BasicBlock *, 64>; bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, const TargetLibraryInfo &RunTLI, AAResults &RunAA, @@ -192,17 +246,20 @@ private: bool processLoad(LoadInst *L); bool processNonLocalLoad(LoadInst *L); bool processAssumeIntrinsic(IntrinsicInst *II); + /// Given a local dependency (Def or Clobber) determine if a value is /// available for the load. Returns true if an value is known to be /// available and populates Res. Returns false otherwise. bool AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo, Value *Address, gvn::AvailableValue &Res); + /// Given a list of non-local dependencies, determine if a value is /// available for the load in each specified block. If it is, add it to /// ValuesPerBlock. If not, add it to UnavailableBlocks. void AnalyzeLoadAvailability(LoadInst *LI, LoadDepVect &Deps, AvailValInBlkVect &ValuesPerBlock, UnavailBlkVect &UnavailableBlocks); + bool PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock, UnavailBlkVect &UnavailableBlocks); @@ -214,9 +271,10 @@ private: bool performPRE(Function &F); bool performScalarPRE(Instruction *I); bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, - unsigned int ValNo); + BasicBlock *Curr, unsigned int ValNo); Value *findLeader(const BasicBlock *BB, uint32_t num); void cleanupGlobalSets(); + void fillImplicitControlFlowInfo(BasicBlock *BB); void verifyRemoved(const Instruction *I) const; bool splitCriticalEdges(); BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ); @@ -226,6 +284,7 @@ private: bool processFoldableCondBr(BranchInst *BI); void addDeadBlock(BasicBlock *BB); void assignValNumForDeadCode(); + void assignBlockRPONumber(Function &F); }; /// Create a legacy GVN pass. This also allows parameterizing whether or not @@ -238,12 +297,14 @@ struct GVNHoistPass : PassInfoMixin<GVNHoistPass> { /// \brief Run the pass over the function. PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + /// \brief Uses an "inverted" value numbering to decide the similarity of /// expressions and sinks similar expressions into successors. struct GVNSinkPass : PassInfoMixin<GVNSinkPass> { /// \brief Run the pass over the function. PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} -#endif +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_GVN_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/GVNExpression.h b/contrib/llvm/include/llvm/Transforms/Scalar/GVNExpression.h index f603ebcbca7c..99dae15a3ac0 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/GVNExpression.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/GVNExpression.h @@ -1,4 +1,4 @@ -//======- GVNExpression.h - GVN Expression classes --------------*- C++ -*-===// +//===- GVNExpression.h - GVN Expression classes -----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,11 +6,12 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// /// The header file for the GVN pass that contains expression handling /// classes -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_GVNEXPRESSION_H @@ -25,7 +26,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> @@ -34,6 +35,9 @@ namespace llvm { +class BasicBlock; +class Type; + namespace GVNExpression { enum ExpressionType { @@ -58,17 +62,18 @@ class Expression { private: ExpressionType EType; unsigned Opcode; - mutable hash_code HashVal; + mutable hash_code HashVal = 0; public: Expression(ExpressionType ET = ET_Base, unsigned O = ~2U) - : EType(ET), Opcode(O), HashVal(0) {} + : EType(ET), Opcode(O) {} Expression(const Expression &) = delete; Expression &operator=(const Expression &) = delete; virtual ~Expression(); static unsigned getEmptyKey() { return ~0U; } static unsigned getTombstoneKey() { return ~1U; } + bool operator!=(const Expression &Other) const { return !(*this == Other); } bool operator==(const Expression &Other) const { if (getOpcode() != Other.getOpcode()) @@ -83,6 +88,7 @@ public: return equals(Other); } + hash_code getComputedHash() const { // It's theoretically possible for a thing to hash to zero. In that case, // we will just compute the hash a few extra times, which is no worse that @@ -93,6 +99,7 @@ public: } virtual bool equals(const Expression &Other) const { return true; } + // Return true if the two expressions are exactly the same, including the // normally ignored fields. virtual bool exactlyEquals(const Expression &Other) const { @@ -106,9 +113,7 @@ public: // We deliberately leave the expression type out of the hash value. virtual hash_code getHashValue() const { return getOpcode(); } - // // Debugging support - // virtual void printInternal(raw_ostream &OS, bool PrintEType) const { if (PrintEType) OS << "etype = " << getExpressionType() << ","; @@ -131,19 +136,19 @@ inline raw_ostream &operator<<(raw_ostream &OS, const Expression &E) { class BasicExpression : public Expression { private: - typedef ArrayRecycler<Value *> RecyclerType; - typedef RecyclerType::Capacity RecyclerCapacity; - Value **Operands; + using RecyclerType = ArrayRecycler<Value *>; + using RecyclerCapacity = RecyclerType::Capacity; + + Value **Operands = nullptr; unsigned MaxOperands; - unsigned NumOperands; - Type *ValueType; + unsigned NumOperands = 0; + Type *ValueType = nullptr; public: BasicExpression(unsigned NumOperands) : BasicExpression(NumOperands, ET_Basic) {} BasicExpression(unsigned NumOperands, ExpressionType ET) - : Expression(ET), Operands(nullptr), MaxOperands(NumOperands), - NumOperands(0), ValueType(nullptr) {} + : Expression(ET), MaxOperands(NumOperands) {} BasicExpression() = delete; BasicExpression(const BasicExpression &) = delete; BasicExpression &operator=(const BasicExpression &) = delete; @@ -174,8 +179,9 @@ public: unsigned getNumOperands() const { return NumOperands; } - typedef Value **op_iterator; - typedef Value *const *const_op_iterator; + using op_iterator = Value **; + using const_op_iterator = Value *const *; + op_iterator op_begin() { return Operands; } op_iterator op_end() { return Operands + NumOperands; } const_op_iterator op_begin() const { return Operands; } @@ -219,9 +225,7 @@ public: hash_combine_range(op_begin(), op_end())); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeBasic, "; @@ -240,7 +244,8 @@ public: class op_inserter : public std::iterator<std::output_iterator_tag, void, void, void, void> { private: - typedef BasicExpression Container; + using Container = BasicExpression; + Container *BE; public: @@ -263,15 +268,16 @@ private: public: MemoryExpression(unsigned NumOperands, enum ExpressionType EType, const MemoryAccess *MemoryLeader) - : BasicExpression(NumOperands, EType), MemoryLeader(MemoryLeader){}; - + : BasicExpression(NumOperands, EType), MemoryLeader(MemoryLeader) {} MemoryExpression() = delete; MemoryExpression(const MemoryExpression &) = delete; MemoryExpression &operator=(const MemoryExpression &) = delete; + static bool classof(const Expression *EB) { return EB->getExpressionType() > ET_MemoryStart && EB->getExpressionType() < ET_MemoryEnd; } + hash_code getHashValue() const override { return hash_combine(this->BasicExpression::getHashValue(), MemoryLeader); } @@ -305,9 +311,7 @@ public: return EB->getExpressionType() == ET_Call; } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeCall, "; @@ -326,11 +330,13 @@ public: LoadExpression(unsigned NumOperands, LoadInst *L, const MemoryAccess *MemoryLeader) : LoadExpression(ET_Load, NumOperands, L, MemoryLeader) {} + LoadExpression(enum ExpressionType EType, unsigned NumOperands, LoadInst *L, const MemoryAccess *MemoryLeader) : MemoryExpression(NumOperands, EType, MemoryLeader), Load(L) { Alignment = L ? L->getAlignment() : 0; } + LoadExpression() = delete; LoadExpression(const LoadExpression &) = delete; LoadExpression &operator=(const LoadExpression &) = delete; @@ -352,9 +358,7 @@ public: cast<LoadExpression>(Other).getLoadInst() == getLoadInst(); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeLoad, "; @@ -388,13 +392,13 @@ public: Value *getStoredValue() const { return StoredValue; } bool equals(const Expression &Other) const override; + bool exactlyEquals(const Expression &Other) const override { return Expression::exactlyEquals(Other) && cast<StoreExpression>(Other).getStoreInst() == getStoreInst(); } // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeStore, "; @@ -409,14 +413,13 @@ public: class AggregateValueExpression final : public BasicExpression { private: unsigned MaxIntOperands; - unsigned NumIntOperands; - unsigned *IntOperands; + unsigned NumIntOperands = 0; + unsigned *IntOperands = nullptr; public: AggregateValueExpression(unsigned NumOperands, unsigned NumIntOperands) : BasicExpression(NumOperands, ET_AggregateValue), - MaxIntOperands(NumIntOperands), NumIntOperands(0), - IntOperands(nullptr) {} + MaxIntOperands(NumIntOperands) {} AggregateValueExpression() = delete; AggregateValueExpression(const AggregateValueExpression &) = delete; AggregateValueExpression & @@ -427,8 +430,8 @@ public: return EB->getExpressionType() == ET_AggregateValue; } - typedef unsigned *int_arg_iterator; - typedef const unsigned *const_int_arg_iterator; + using int_arg_iterator = unsigned *; + using const_int_arg_iterator = const unsigned *; int_arg_iterator int_op_begin() { return IntOperands; } int_arg_iterator int_op_end() { return IntOperands + NumIntOperands; } @@ -463,9 +466,7 @@ public: hash_combine_range(int_op_begin(), int_op_end())); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeAggregateValue, "; @@ -481,7 +482,8 @@ public: class int_op_inserter : public std::iterator<std::output_iterator_tag, void, void, void, void> { private: - typedef AggregateValueExpression Container; + using Container = AggregateValueExpression; + Container *AVE; public: @@ -524,9 +526,7 @@ public: return hash_combine(this->BasicExpression::getHashValue(), BB); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypePhi, "; @@ -573,9 +573,7 @@ public: VariableValue->getType(), VariableValue); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeVariable, "; @@ -612,9 +610,7 @@ public: ConstantValue->getType(), ConstantValue); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeConstant, "; @@ -649,9 +645,7 @@ public: return hash_combine(this->Expression::getHashValue(), Inst); } - // // Debugging support - // void printInternal(raw_ostream &OS, bool PrintEType) const override { if (PrintEType) OS << "ExpressionTypeUnknown, "; diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/IndVarSimplify.h b/contrib/llvm/include/llvm/Transforms/Scalar/IndVarSimplify.h index 4a4683f1a07d..e321c8fc6e9c 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/IndVarSimplify.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/IndVarSimplify.h @@ -15,17 +15,20 @@ #ifndef LLVM_TRANSFORMS_SCALAR_INDVARSIMPLIFY_H #define LLVM_TRANSFORMS_SCALAR_INDVARSIMPLIFY_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Loop; +class LPMUpdater; + class IndVarSimplifyPass : public PassInfoMixin<IndVarSimplifyPass> { public: PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_INDVARSIMPLIFY_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/contrib/llvm/include/llvm/Transforms/Scalar/JumpThreading.h index 1da86132591b..a9466713b8e6 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/JumpThreading.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/JumpThreading.h @@ -6,41 +6,57 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// See the comments on JumpThreadingPass. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_JUMPTHREADING_H #define LLVM_TRANSFORMS_SCALAR_JUMPTHREADING_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/BlockFrequencyInfoImpl.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/LazyValueInfo.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/ValueHandle.h" +#include <memory> +#include <utility> namespace llvm { +class BasicBlock; +class BinaryOperator; +class BranchInst; +class CmpInst; +class Constant; +class Function; +class Instruction; +class IntrinsicInst; +class LazyValueInfo; +class LoadInst; +class PHINode; +class TargetLibraryInfo; +class Value; + /// A private "module" namespace for types and utilities used by /// JumpThreading. /// These are implementation details and should not be used by clients. namespace jumpthreading { + // These are at global scope so static functions can use them too. -typedef SmallVectorImpl<std::pair<Constant *, BasicBlock *>> PredValueInfo; -typedef SmallVector<std::pair<Constant *, BasicBlock *>, 8> PredValueInfoTy; +using PredValueInfo = SmallVectorImpl<std::pair<Constant *, BasicBlock *>>; +using PredValueInfoTy = SmallVector<std::pair<Constant *, BasicBlock *>, 8>; // This is used to keep track of what kind of constant we're currently hoping // to find. enum ConstantPreference { WantInteger, WantBlockAddress }; -} + +} // end namespace jumpthreading /// This pass performs 'jump threading', which looks at blocks that have /// multiple predecessors and multiple successors. If one or more of the @@ -57,7 +73,6 @@ enum ConstantPreference { WantInteger, WantBlockAddress }; /// /// In this case, the unconditional branch at the end of the first if can be /// revectored to the false side of the second if. -/// class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> { TargetLibraryInfo *TLI; LazyValueInfo *LVI; @@ -141,4 +156,4 @@ private: } // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_SCALAR_JUMPTHREADING_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopDistribute.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopDistribute.h index ddde5954c218..2bf1c9d696d5 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopDistribute.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopDistribute.h @@ -21,10 +21,13 @@ namespace llvm { +class Function; + class LoopDistributePass : public PassInfoMixin<LoopDistributePass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPDISTRIBUTE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h index 40349e8f7fe0..7added8d2c61 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h @@ -1,4 +1,4 @@ -//===- LoopIdiomRecognize.h - Loop Idiom Recognize Pass -------*- C++ -*-===// +//===- LoopIdiomRecognize.h - Loop Idiom Recognize Pass ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,18 +16,21 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPIDIOMRECOGNIZE_H #define LLVM_TRANSFORMS_SCALAR_LOOPIDIOMRECOGNIZE_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Loop; +class LPMUpdater; + /// Performs Loop Idiom Recognize Pass. class LoopIdiomRecognizePass : public PassInfoMixin<LoopIdiomRecognizePass> { public: PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPIDIOMRECOGNIZE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopInstSimplify.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopInstSimplify.h index bb8bc29577a2..04dc79c3fa57 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopInstSimplify.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopInstSimplify.h @@ -1,4 +1,4 @@ -//===- LoopInstSimplify.h - Loop Inst Simplify Pass -------*- C++ -*-===// +//===- LoopInstSimplify.h - Loop Inst Simplify Pass -------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,18 +14,21 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPINSTSIMPLIFY_H #define LLVM_TRANSFORMS_SCALAR_LOOPINSTSIMPLIFY_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Loop; +class LPMUpdater; + /// Performs Loop Inst Simplify Pass. class LoopInstSimplifyPass : public PassInfoMixin<LoopInstSimplifyPass> { public: PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPINSTSIMPLIFY_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopLoadElimination.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopLoadElimination.h index 7a007a7e822d..b0514a4a7c98 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopLoadElimination.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopLoadElimination.h @@ -1,4 +1,4 @@ -//===---- LoopLoadElimination.h ---------------------------------*- C++ -*-===// +//===- LoopLoadElimination.h ------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,11 +6,12 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This header defines the LoopLoadEliminationPass object. This pass forwards /// loaded values around loop backedges to allow their use in subsequent /// iterations. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_LOOPLOADELIMINATION_H @@ -20,11 +21,14 @@ namespace llvm { +class Function; + /// Pass to forward loads in a loop around the backedge to subsequent /// iterations. struct LoopLoadEliminationPass : public PassInfoMixin<LoopLoadEliminationPass> { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPLOADELIMINATION_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h index 715b11d3d974..473b97dc7e8d 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -164,10 +164,11 @@ public: /// If this is called for the current loop, in addition to clearing any /// state, this routine will mark that the current loop should be skipped by /// the rest of the pass management infrastructure. - void markLoopAsDeleted(Loop &L) { - LAM.clear(L); - assert(CurrentL->contains(&L) && "Cannot delete a loop outside of the " - "subloop tree currently being processed."); + void markLoopAsDeleted(Loop &L, llvm::StringRef Name) { + LAM.clear(L, Name); + assert((&L == CurrentL || CurrentL->contains(&L)) && + "Cannot delete a loop outside of the " + "subloop tree currently being processed."); if (&L == CurrentL) SkipCurrentLoop = true; } @@ -216,6 +217,19 @@ public: // shouldn't impact anything. } + /// Restart the current loop. + /// + /// Loop passes should call this method to indicate the current loop has been + /// sufficiently changed that it should be re-visited from the begining of + /// the loop pass pipeline rather than continuing. + void revisitCurrentLoop() { + // Tell the currently in-flight pipeline to stop running. + SkipCurrentLoop = true; + + // And insert ourselves back into the worklist. + Worklist.insert(CurrentL); + } + private: template <typename LoopPassT> friend class llvm::FunctionToLoopPassAdaptor; @@ -271,13 +285,17 @@ public: return PA; // Get the analysis results needed by loop passes. + MemorySSA *MSSA = EnableMSSALoopDependency + ? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA()) + : nullptr; LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F), AM.getResult<AssumptionAnalysis>(F), AM.getResult<DominatorTreeAnalysis>(F), AM.getResult<LoopAnalysis>(F), AM.getResult<ScalarEvolutionAnalysis>(F), AM.getResult<TargetLibraryAnalysis>(F), - AM.getResult<TargetIRAnalysis>(F)}; + AM.getResult<TargetIRAnalysis>(F), + MSSA}; // Setup the loop analysis manager from its proxy. It is important that // this is only done when there are loops to process and we have built the @@ -345,6 +363,8 @@ public: PA.preserve<DominatorTreeAnalysis>(); PA.preserve<LoopAnalysis>(); PA.preserve<ScalarEvolutionAnalysis>(); + // FIXME: Uncomment this when all loop passes preserve MemorySSA + // PA.preserve<MemorySSAAnalysis>(); // FIXME: What we really want to do here is preserve an AA category, but // that concept doesn't exist yet. PA.preserve<AAManager>(); diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopStrengthReduce.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopStrengthReduce.h index ebcb32125262..62c038a3857d 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopStrengthReduce.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopStrengthReduce.h @@ -1,4 +1,4 @@ -//===- LoopStrengthReduce.h - Loop Strength Reduce Pass -------*- C++ -*-===// +//===- LoopStrengthReduce.h - Loop Strength Reduce Pass ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -22,18 +22,21 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPSTRENGTHREDUCE_H #define LLVM_TRANSFORMS_SCALAR_LOOPSTRENGTHREDUCE_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { +class Loop; +class LPMUpdater; + /// Performs Loop Strength Reduce Pass. class LoopStrengthReducePass : public PassInfoMixin<LoopStrengthReducePass> { public: PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPSTRENGTHREDUCE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/contrib/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h index 7253bd09766e..9848e0d54f2b 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h @@ -10,40 +10,40 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOOPUNROLLPASS_H #define LLVM_TRANSFORMS_SCALAR_LOOPUNROLLPASS_H -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { -class LoopUnrollPass : public PassInfoMixin<LoopUnrollPass> { - const bool AllowPartialUnrolling; +class Function; +class Loop; +class LPMUpdater; + +/// Loop unroll pass that only does full loop unrolling. +class LoopFullUnrollPass : public PassInfoMixin<LoopFullUnrollPass> { const int OptLevel; - explicit LoopUnrollPass(bool AllowPartialUnrolling, int OptLevel) - : AllowPartialUnrolling(AllowPartialUnrolling), OptLevel(OptLevel) {} +public: + explicit LoopFullUnrollPass(int OptLevel = 2) : OptLevel(OptLevel) {} + + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); +}; + +/// Loop unroll pass that will support both full and partial unrolling. +/// It is a function pass to have access to function and module analyses. +/// It will also put loops into canonical form (simplified and LCSSA). +class LoopUnrollPass : public PassInfoMixin<LoopUnrollPass> { + const int OptLevel; public: - /// Create an instance of the loop unroll pass that will support both full - /// and partial unrolling. - /// /// This uses the target information (or flags) to control the thresholds for /// different unrolling stategies but supports all of them. - static LoopUnrollPass create(int OptLevel = 2) { - return LoopUnrollPass(/*AllowPartialUnrolling*/ true, OptLevel); - } - - /// Create an instance of the loop unroll pass that only does full loop - /// unrolling. - /// - /// This will disable any runtime or partial unrolling. - static LoopUnrollPass createFull(int OptLevel = 2) { - return LoopUnrollPass(/*AllowPartialUnrolling*/ false, OptLevel); - } + explicit LoopUnrollPass(int OptLevel = 2) : OptLevel(OptLevel) {} - PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, - LoopStandardAnalysisResults &AR, LPMUpdater &U); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_LOOPUNROLLPASS_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/contrib/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h index f688e7f19986..ab9dec0311b2 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h @@ -24,7 +24,7 @@ namespace llvm { struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> { /// \brief Run the pass over the function. /// - /// This will lower all of th expect intrinsic calls in this function into + /// This will lower all of the expect intrinsic calls in this function into /// branch weight metadata. That metadata will subsequently feed the analysis /// of the probabilities and frequencies of the CFG. After running this pass, /// no more expect intrinsics remain, allowing the rest of the optimizer to diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/contrib/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h index f52872dd2ea7..046c808bd051 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h @@ -1,4 +1,4 @@ -//===---- MemCpyOptimizer.h - memcpy optimization ---------------*- C++ -*-===// +//===- MemCpyOptimizer.h - memcpy optimization ------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,20 +16,27 @@ #define LLVM_TRANSFORMS_SCALAR_MEMCPYOPTIMIZER_H #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/MemoryDependenceAnalysis.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/PassManager.h" #include <cstdint> #include <functional> namespace llvm { +class AssumptionCache; +class CallInst; +class DominatorTree; +class Function; +class Instruction; +class MemCpyInst; +class MemMoveInst; +class MemoryDependenceResults; +class MemSetInst; +class StoreInst; +class TargetLibraryInfo; +class Value; + class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> { MemoryDependenceResults *MD = nullptr; TargetLibraryInfo *TLI = nullptr; @@ -41,6 +48,7 @@ public: MemCpyOptPass() = default; PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + // Glue for the old PM. bool runImpl(Function &F, MemoryDependenceResults *MD_, TargetLibraryInfo *TLI_, diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h b/contrib/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h index f35707eeb3f0..e835bd5f0761 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h @@ -1,4 +1,4 @@ -//===- NaryReassociate.h - Reassociate n-ary expressions ------------------===// +//===- NaryReassociate.h - Reassociate n-ary expressions --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -81,15 +81,25 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ValueHandle.h" namespace llvm { + +class AssumptionCache; +class BinaryOperator; +class DataLayout; +class DominatorTree; +class Function; +class GetElementPtrInst; +class Instruction; +class ScalarEvolution; +class SCEV; +class TargetLibraryInfo; +class TargetTransformInfo; +class Type; +class Value; + class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -109,6 +119,7 @@ private: // Reassociate GEP for better CSE. Instruction *tryReassociateGEP(GetElementPtrInst *GEP); + // Try splitting GEP at the I-th index and see whether either part can be // CSE'ed. This is a helper function for tryReassociateGEP. // @@ -118,6 +129,7 @@ private: // ..., i-th index). GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, unsigned I, Type *IndexedType); + // Given GEP's I-th index = LHS + RHS, see whether &Base[..][LHS][..] or // &Base[..][RHS][..] can be CSE'ed and rewrite GEP accordingly. GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, @@ -146,6 +158,7 @@ private: // \c CandidateExpr. Returns null if not found. Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr, Instruction *Dominatee); + // GetElementPtrInst implicitly sign-extends an index if the index is shorter // than the pointer size. This function returns whether Index is shorter than // GEP's pointer size, i.e., whether Index needs to be sign-extended in order @@ -158,6 +171,7 @@ private: ScalarEvolution *SE; TargetLibraryInfo *TLI; TargetTransformInfo *TTI; + // A lookup table quickly telling which instructions compute the given SCEV. // Note that there can be multiple instructions at different locations // computing to the same SCEV, so we map a SCEV to an instruction list. For @@ -169,6 +183,7 @@ private: // bar(a + b); DenseMap<const SCEV *, SmallVector<WeakTrackingVH, 2>> SeenExprs; }; -} // namespace llvm + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_NARYREASSOCIATE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/NewGVN.h b/contrib/llvm/include/llvm/Transforms/Scalar/NewGVN.h index d0425aa4345f..05db25502dc3 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/NewGVN.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/NewGVN.h @@ -1,4 +1,4 @@ -//===----- NewGVN.h - Global Value Numbering Pass ---------------*- C++ -*-===// +//===- NewGVN.h - Global Value Numbering Pass -------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,9 +6,10 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides the interface for LLVM's Global Value Numbering pass. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_NEWGVN_H @@ -17,12 +18,16 @@ #include "llvm/IR/PassManager.h" namespace llvm { + +class Function; + class NewGVNPass : public PassInfoMixin<NewGVNPass> { public: /// \brief Run the pass over the function. PreservedAnalyses run(Function &F, AnalysisManager<Function> &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_NEWGVN_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/contrib/llvm/include/llvm/Transforms/Scalar/Reassociate.h index a30a7176baa8..9997dfa5b6f3 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/Reassociate.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/Reassociate.h @@ -23,22 +23,33 @@ #ifndef LLVM_TRANSFORMS_SCALAR_REASSOCIATE_H #define LLVM_TRANSFORMS_SCALAR_REASSOCIATE_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ValueHandle.h" namespace llvm { +class APInt; +class BasicBlock; +class BinaryOperator; +class Function; +class Instruction; +class Value; + /// A private "module" namespace for types and utilities used by Reassociate. /// These are implementation details and should not be used by clients. namespace reassociate { + struct ValueEntry { unsigned Rank; Value *Op; + ValueEntry(unsigned R, Value *O) : Rank(R), Op(O) {} }; + inline bool operator<(const ValueEntry &LHS, const ValueEntry &RHS) { return LHS.Rank > RHS.Rank; // Sort so that highest rank goes to start. } @@ -48,17 +59,26 @@ inline bool operator<(const ValueEntry &LHS, const ValueEntry &RHS) { struct Factor { Value *Base; unsigned Power; + Factor(Value *Base, unsigned Power) : Base(Base), Power(Power) {} }; class XorOpnd; -} + +} // end namespace reassociate /// Reassociate commutative expressions. class ReassociatePass : public PassInfoMixin<ReassociatePass> { DenseMap<BasicBlock *, unsigned> RankMap; DenseMap<AssertingVH<Value>, unsigned> ValueRankMap; SetVector<AssertingVH<Instruction>> RedoInsts; + + // Arbitrary, but prevents quadratic behavior. + static const unsigned GlobalReassociateLimit = 10; + static const unsigned NumBinaryOps = + Instruction::BinaryOpsEnd - Instruction::BinaryOpsBegin; + DenseMap<std::pair<Value *, Value *>, unsigned> PairMap[NumBinaryOps]; + bool MadeChange; public: @@ -92,7 +112,9 @@ private: SetVector<AssertingVH<Instruction>> &Insts); void OptimizeInst(Instruction *I); Instruction *canonicalizeNegConstExpr(Instruction *I); + void BuildPairMap(ReversePostOrderTraversal<Function *> &RPOT); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_REASSOCIATE_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h b/contrib/llvm/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h new file mode 100644 index 000000000000..128f176f4420 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h @@ -0,0 +1,39 @@ +//===- RewriteStatepointsForGC.h - ------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides interface to "Rewrite Statepoints for GC" pass. +// +// This passe rewrites call/invoke instructions so as to make potential +// relocations performed by the garbage collector explicit in the IR. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_REWRITE_STATEPOINTS_FOR_GC_H +#define LLVM_TRANSFORMS_SCALAR_REWRITE_STATEPOINTS_FOR_GC_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class DominatorTree; +class Function; +class Module; +class TargetTransformInfo; +class TargetLibraryInfo; + +struct RewriteStatepointsForGC : public PassInfoMixin<RewriteStatepointsForGC> { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + + bool runOnFunction(Function &F, DominatorTree &, TargetTransformInfo &, + const TargetLibraryInfo &); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_REWRITE_STATEPOINTS_FOR_GC_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/SCCP.h b/contrib/llvm/include/llvm/Transforms/Scalar/SCCP.h index 6e7f77fe2c50..b93287fff907 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/SCCP.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/SCCP.h @@ -6,7 +6,8 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -/// \file +// +// \file // This file implements sparse conditional constant propagation and merging: // // Specifically, this: @@ -15,22 +16,23 @@ // * Proves values to be constant, and replaces them with constants // * Proves conditional branches to be unconditional // -/// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_SCALAR_SCCP_H #define LLVM_TRANSFORMS_SCALAR_SCCP_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { +class Function; + /// This pass performs function-level constant propagation and merging. class SCCPPass : public PassInfoMixin<SCCPPass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_SCCP_H diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/SROA.h b/contrib/llvm/include/llvm/Transforms/Scalar/SROA.h index 3080b75ba894..4a321e75c68b 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -17,15 +17,23 @@ #define LLVM_TRANSFORMS_SCALAR_SROA_H #include "llvm/ADT/SetVector.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Compiler.h" #include <vector> namespace llvm { +class AllocaInst; +class AssumptionCache; +class DominatorTree; +class Function; +class Instruction; +class LLVMContext; +class PHINode; +class SelectInst; +class Use; + /// A private "module" namespace for types and utilities used by SROA. These /// are implementation details and should not be used by clients. namespace sroa LLVM_LIBRARY_VISIBILITY { @@ -122,7 +130,7 @@ private: bool splitAlloca(AllocaInst &AI, sroa::AllocaSlices &AS); bool runOnAlloca(AllocaInst &AI); void clobberUse(Use &U); - void deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas); + bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas); bool promoteAllocas(Function &F); }; diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/contrib/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h index d7282ac6a781..63bfe6373d04 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h @@ -36,8 +36,10 @@ namespace llvm { /// of the loop, to make the unswitching opportunity obvious. /// class SimpleLoopUnswitchPass : public PassInfoMixin<SimpleLoopUnswitchPass> { + bool NonTrivial; + public: - SimpleLoopUnswitchPass() = default; + SimpleLoopUnswitchPass(bool NonTrivial = false) : NonTrivial(NonTrivial) {} PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); @@ -46,7 +48,7 @@ public: /// Create the legacy pass object for the simple loop unswitcher. /// /// See the documentaion for `SimpleLoopUnswitchPass` for details. -Pass *createSimpleLoopUnswitchLegacyPass(); +Pass *createSimpleLoopUnswitchLegacyPass(bool NonTrivial = false); } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h b/contrib/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h index 54b51c405ad4..1afb9c7f954f 100644 --- a/contrib/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h +++ b/contrib/llvm/include/llvm/Transforms/Scalar/SimplifyCFG.h @@ -17,26 +17,34 @@ #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Utils/Local.h" namespace llvm { -/// \brief A pass to simplify and canonicalize the CFG of a function. +/// A pass to simplify and canonicalize the CFG of a function. /// -/// This pass iteratively simplifies the entire CFG of a function, removing -/// unnecessary control flows and bringing it into the canonical form expected -/// by the rest of the mid-level optimizer. +/// This pass iteratively simplifies the entire CFG of a function. It may change +/// or remove control flow to put the CFG into a canonical form expected by +/// other passes of the mid-level optimizer. Depending on the specified options, +/// it may further optimize control-flow to create non-canonical forms. class SimplifyCFGPass : public PassInfoMixin<SimplifyCFGPass> { - int BonusInstThreshold; - bool LateSimplifyCFG; + SimplifyCFGOptions Options; public: - /// \brief Construct a pass with the default thresholds - /// and switch optimizations. - SimplifyCFGPass(); - - /// \brief Construct a pass with a specific bonus threshold - /// and optional switch optimizations. - SimplifyCFGPass(int BonusInstThreshold, bool LateSimplifyCFG); + /// The default constructor sets the pass options to create canonical IR, + /// rather than optimal IR. That is, by default we bypass transformations that + /// are likely to improve performance but make analysis for other passes more + /// difficult. + SimplifyCFGPass() + : SimplifyCFGPass(SimplifyCFGOptions() + .forwardSwitchCondToPhi(false) + .convertSwitchToLookupTable(false) + .needCanonicalLoops(true) + .sinkCommonInsts(false)) {} + + + /// Construct a pass with optional optimizations. + SimplifyCFGPass(const SimplifyCFGOptions &PassOptions); /// \brief Run the pass over the function. PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); diff --git a/contrib/llvm/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h b/contrib/llvm/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h new file mode 100644 index 000000000000..f39e03d22d65 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h @@ -0,0 +1,111 @@ +//===- SpeculateAroundPHIs.h - Speculate around PHIs ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_SPECULATEAROUNDPHIS_H +#define LLVM_TRANSFORMS_SCALAR_SPECULATEAROUNDPHIS_H + +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" +#include <vector> + +namespace llvm { + +/// This pass handles simple speculating of instructions around PHIs when +/// doing so is profitable for a particular target despite duplicated +/// instructions. +/// +/// The motivating example are PHIs of constants which will require +/// materializing the constants along each edge. If the PHI is used by an +/// instruction where the target can materialize the constant as part of the +/// instruction, it is profitable to speculate those instructions around the +/// PHI node. This can reduce dynamic instruction count as well as decrease +/// register pressure. +/// +/// Consider this IR for example: +/// ``` +/// entry: +/// br i1 %flag, label %a, label %b +/// +/// a: +/// br label %exit +/// +/// b: +/// br label %exit +/// +/// exit: +/// %p = phi i32 [ 7, %a ], [ 11, %b ] +/// %sum = add i32 %arg, %p +/// ret i32 %sum +/// ``` +/// To materialize the inputs to this PHI node may require an explicit +/// instruction. For example, on x86 this would turn into something like +/// ``` +/// testq %eax, %eax +/// movl $7, %rNN +/// jne .L +/// movl $11, %rNN +/// .L: +/// addl %edi, %rNN +/// movl %rNN, %eax +/// retq +/// ``` +/// When these constants can be folded directly into another instruction, it +/// would be preferable to avoid the potential for register pressure (above we +/// can easily avoid it, but that isn't always true) and simply duplicate the +/// instruction using the PHI: +/// ``` +/// entry: +/// br i1 %flag, label %a, label %b +/// +/// a: +/// %sum.1 = add i32 %arg, 7 +/// br label %exit +/// +/// b: +/// %sum.2 = add i32 %arg, 11 +/// br label %exit +/// +/// exit: +/// %p = phi i32 [ %sum.1, %a ], [ %sum.2, %b ] +/// ret i32 %p +/// ``` +/// Which will generate something like the following on x86: +/// ``` +/// testq %eax, %eax +/// addl $7, %edi +/// jne .L +/// addl $11, %edi +/// .L: +/// movl %edi, %eax +/// retq +/// ``` +/// +/// It is important to note that this pass is never intended to handle more +/// complex cases where speculating around PHIs allows simplifications of the +/// IR itself or other subsequent optimizations. Those can and should already +/// be handled before this pass is ever run by a more powerful analysis that +/// can reason about equivalences and common subexpressions. Classically, those +/// cases would be handled by a GVN-powered PRE or similar transform. This +/// pass, in contrast, is *only* interested in cases where despite no +/// simplifications to the IR itself, speculation is *faster* to execute. The +/// result of this is that the cost models which are appropriate to consider +/// here are relatively simple ones around execution and codesize cost, without +/// any need to consider simplifications or other transformations. +struct SpeculateAroundPHIsPass : PassInfoMixin<SpeculateAroundPHIsPass> { + /// \brief Run the pass over the function. + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_SPECULATEAROUNDPHIS_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h b/contrib/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h index a87758300992..4dad06e6c125 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h @@ -1,4 +1,4 @@ -//===- AddDiscriminators.h -------------------------------------*- C++ -*-===// +//===- AddDiscriminators.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -20,10 +20,13 @@ namespace llvm { +class Function; + class AddDiscriminatorsPass : public PassInfoMixin<AddDiscriminatorsPass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_ADDDISCRIMINATORS_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index 85bb053135a6..74f75509f550 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -1,4 +1,4 @@ -//===-- Transform/Utils/BasicBlockUtils.h - BasicBlock Utils ----*- C++ -*-===// +//===- Transform/Utils/BasicBlockUtils.h - BasicBlock Utils -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -25,13 +25,17 @@ namespace llvm { -class MemoryDependenceResults; +class BlockFrequencyInfo; +class BranchProbabilityInfo; class DominatorTree; -class LoopInfo; +class Function; class Instruction; +class LoopInfo; class MDNode; +class MemoryDependenceResults; class ReturnInst; class TargetLibraryInfo; +class Value; /// Delete the specified block, which must have no predecessors. void DeleteDeadBlock(BasicBlock *BB); @@ -118,7 +122,6 @@ struct CriticalEdgeSplittingOptions { /// IndirectBrInst. Splitting these edges will almost always create an invalid /// program because the address of the new block won't be the one that is jumped /// to. -/// BasicBlock *SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options = CriticalEdgeSplittingOptions()); @@ -194,7 +197,6 @@ BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, /// no other analyses. In particular, it does not preserve LoopSimplify /// (because it's complicated to handle the case where one of the edges being /// split is an exit of a loop with other exits). -/// BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix, DominatorTree *DT = nullptr, @@ -212,7 +214,6 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds, /// no other analyses. In particular, it does not preserve LoopSimplify /// (because it's complicated to handle the case where one of the edges being /// split is an exit of a loop with other exits). -/// void SplitLandingPadPredecessors(BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix, const char *Suffix2, @@ -284,6 +285,29 @@ void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, Value *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, BasicBlock *&IfFalse); +// Split critical edges where the source of the edge is an indirectbr +// instruction. This isn't always possible, but we can handle some easy cases. +// This is useful because MI is unable to split such critical edges, +// which means it will not be able to sink instructions along those edges. +// This is especially painful for indirect branches with many successors, where +// we end up having to prepare all outgoing values in the origin block. +// +// Our normal algorithm for splitting critical edges requires us to update +// the outgoing edges of the edge origin block, but for an indirectbr this +// is hard, since it would require finding and updating the block addresses +// the indirect branch uses. But if a block only has a single indirectbr +// predecessor, with the others being regular branches, we can do it in a +// different way. +// Say we have A -> D, B -> D, I -> D where only I -> D is an indirectbr. +// We can split D into D0 and D1, where D0 contains only the PHIs from D, +// and D1 is the D block body. We can then duplicate D0 as D0A and D0B, and +// create the following structure: +// A -> D0A, B -> D0A, I -> D0B, D0A -> D1, D0B -> D1 +// If BPI and BFI aren't non-null, BPI/BFI will be updated accordingly. +bool SplitIndirectBrCriticalEdges(Function &F, + BranchProbabilityInfo *BPI = nullptr, + BlockFrequencyInfo *BFI = nullptr); + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_BASICBLOCKUTILS_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h b/contrib/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h index af0d60b2625f..6eca5ed2154e 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h @@ -1,4 +1,4 @@ -//===- llvm/Transforms/Utils/BypassSlowDivision.h --------------*- C++ -*-===// +//===- llvm/Transforms/Utils/BypassSlowDivision.h ---------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -19,10 +19,44 @@ #define LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H #include "llvm/ADT/DenseMap.h" -#include "llvm/IR/Function.h" +#include "llvm/ADT/DenseMapInfo.h" +#include <cstdint> namespace llvm { +class BasicBlock; +class Value; + +struct DivRemMapKey { + bool SignedOp; + Value *Dividend; + Value *Divisor; + + DivRemMapKey(bool InSignedOp, Value *InDividend, Value *InDivisor) + : SignedOp(InSignedOp), Dividend(InDividend), Divisor(InDivisor) {} +}; + +template <> struct DenseMapInfo<DivRemMapKey> { + static bool isEqual(const DivRemMapKey &Val1, const DivRemMapKey &Val2) { + return Val1.SignedOp == Val2.SignedOp && Val1.Dividend == Val2.Dividend && + Val1.Divisor == Val2.Divisor; + } + + static DivRemMapKey getEmptyKey() { + return DivRemMapKey(false, nullptr, nullptr); + } + + static DivRemMapKey getTombstoneKey() { + return DivRemMapKey(true, nullptr, nullptr); + } + + static unsigned getHashValue(const DivRemMapKey &Val) { + return (unsigned)(reinterpret_cast<uintptr_t>(Val.Dividend) ^ + reinterpret_cast<uintptr_t>(Val.Divisor)) ^ + (unsigned)Val.SignedOp; + } +}; + /// This optimization identifies DIV instructions in a BB that can be /// profitably bypassed and carried out with a shorter, faster divide. /// @@ -31,6 +65,6 @@ namespace llvm { bool bypassSlowDivision( BasicBlock *BB, const DenseMap<unsigned int, unsigned int> &BypassWidth); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h new file mode 100644 index 000000000000..e0bf85781d81 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -0,0 +1,44 @@ +//===- CallPromotionUtils.h - Utilities for call promotion ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares utilities useful for promoting indirect call sites to +// direct call sites. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H +#define LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H + +#include "llvm/IR/CallSite.h" + +namespace llvm { + +/// Return true if the given indirect call site can be made to call \p Callee. +/// +/// This function ensures that the number and type of the call site's arguments +/// and return value match those of the given function. If the types do not +/// match exactly, they must at least be bitcast compatible. If \p FailureReason +/// is non-null and the indirect call cannot be promoted, the failure reason +/// will be stored in it. +bool isLegalToPromote(CallSite CS, Function *Callee, + const char **FailureReason = nullptr); + +/// Promote the given indirect call site to conditionally call \p Callee. +/// +/// This function creates an if-then-else structure at the location of the call +/// site. The original call site is promoted and moved into the "then" block. A +/// clone of the indirect call site is placed in the "else" block and returned. +/// If \p BranchWeights is non-null, it will be used to set !prof metadata on +/// the new conditional branch. +Instruction *promoteCallWithIfThenElse(CallSite CS, Function *Callee, + MDNode *BranchWeights = nullptr); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_CALLPROMOTIONUTILS_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/Cloning.h b/contrib/llvm/include/llvm/Transforms/Utils/Cloning.h index 2a8b89d86282..178bae76cef6 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -227,12 +227,18 @@ public: /// *inlined* code to minimize the actual inserted code, it must not delete /// code in the caller as users of this routine may have pointers to /// instructions in the caller that need to remain stable. +/// +/// If ForwardVarArgsTo is passed, inlining a function with varargs is allowed +/// and all varargs at the callsite will be passed to any calls to +/// ForwardVarArgsTo. The caller of InlineFunction has to make sure any varargs +/// are only used by ForwardVarArgsTo. bool InlineFunction(CallInst *C, InlineFunctionInfo &IFI, AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); bool InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI, AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); bool InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); + AAResults *CalleeAAR = nullptr, bool InsertLifetime = true, + Function *ForwardVarArgsTo = nullptr); /// \brief Clones a loop \p OrigLoop. Returns the loop and the blocks in \p /// Blocks. diff --git a/contrib/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/contrib/llvm/include/llvm/Transforms/Utils/CodeExtractor.h index 682b353ab5ae..63d34511102d 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/CodeExtractor.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/CodeExtractor.h @@ -1,4 +1,4 @@ -//===-- Transform/Utils/CodeExtractor.h - Code extraction util --*- C++ -*-===// +//===- Transform/Utils/CodeExtractor.h - Code extraction util ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,22 +15,24 @@ #ifndef LLVM_TRANSFORMS_UTILS_CODEEXTRACTOR_H #define LLVM_TRANSFORMS_UTILS_CODEEXTRACTOR_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" +#include <limits> namespace llvm { -template <typename T> class ArrayRef; - class BasicBlock; - class BlockFrequency; - class BlockFrequencyInfo; - class BranchProbabilityInfo; - class DominatorTree; - class Function; - class Instruction; - class Loop; - class Module; - class RegionNode; - class Type; - class Value; + +class BasicBlock; +class BlockFrequency; +class BlockFrequencyInfo; +class BranchProbabilityInfo; +class DominatorTree; +class Function; +class Instruction; +class Loop; +class Module; +class Type; +class Value; /// \brief Utility class for extracting code into a new function. /// @@ -46,7 +48,7 @@ template <typename T> class ArrayRef; /// 3) Add allocas for any scalar outputs, adding all of the outputs' allocas /// as arguments, and inserting stores to the arguments for any scalars. class CodeExtractor { - typedef SetVector<Value *> ValueSet; + using ValueSet = SetVector<Value *>; // Various bits of state computed on construction. DominatorTree *const DT; @@ -54,27 +56,27 @@ template <typename T> class ArrayRef; BlockFrequencyInfo *BFI; BranchProbabilityInfo *BPI; + // If true, varargs functions can be extracted. + bool AllowVarArgs; + // Bits of intermediate state computed at various phases of extraction. SetVector<BasicBlock *> Blocks; - unsigned NumExitBlocks; + unsigned NumExitBlocks = std::numeric_limits<unsigned>::max(); Type *RetTy; public: - - /// \brief Check to see if a block is valid for extraction. - /// - /// Blocks containing EHPads, allocas, invokes, or vastarts are not valid. - static bool isBlockValidForExtraction(const BasicBlock &BB); - /// \brief Create a code extractor for a sequence of blocks. /// /// Given a sequence of basic blocks where the first block in the sequence /// dominates the rest, prepare a code extractor object for pulling this /// sequence out into its new function. When a DominatorTree is also given, - /// extra checking and transformations are enabled. + /// extra checking and transformations are enabled. If AllowVarArgs is true, + /// vararg functions can be extracted. This is safe, if all vararg handling + /// code is extracted, including vastart. CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr, bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr, - BranchProbabilityInfo *BPI = nullptr); + BranchProbabilityInfo *BPI = nullptr, + bool AllowVarArgs = false); /// \brief Create a code extractor for a loop body. /// @@ -84,6 +86,14 @@ template <typename T> class ArrayRef; BlockFrequencyInfo *BFI = nullptr, BranchProbabilityInfo *BPI = nullptr); + /// \brief Check to see if a block is valid for extraction. + /// + /// Blocks containing EHPads, allocas and invokes are not valid. If + /// AllowVarArgs is true, blocks with vastart can be extracted. This is + /// safe, if all vararg handling code is extracted, including vastart. + static bool isBlockValidForExtraction(const BasicBlock &BB, + bool AllowVarArgs); + /// \brief Perform the extraction, returning the new function. /// /// Returns zero when called on a CodeExtractor instance where isEligible @@ -112,6 +122,7 @@ template <typename T> class ArrayRef; /// /// Returns true if it is safe to do the code motion. bool isLegalToShrinkwrapLifetimeMarkers(Instruction *AllocaAddr) const; + /// Find the set of allocas whose life ranges are contained within the /// outlined region. /// @@ -155,6 +166,7 @@ template <typename T> class ArrayRef; ValueSet &inputs, ValueSet &outputs); }; -} -#endif +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_CODEEXTRACTOR_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/EntryExitInstrumenter.h b/contrib/llvm/include/llvm/Transforms/Utils/EntryExitInstrumenter.h new file mode 100644 index 000000000000..f50c5c922081 --- /dev/null +++ b/contrib/llvm/include/llvm/Transforms/Utils/EntryExitInstrumenter.h @@ -0,0 +1,36 @@ +//===- EntryExitInstrumenter.h - Function Entry/Exit Instrumentation ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// EntryExitInstrumenter pass - Instrument function entry/exit with calls to +// mcount(), @__cyg_profile_func_{enter,exit} and the like. There are two +// variants, intended to run pre- and post-inlining, respectively. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_ENTRYEXITINSTRUMENTER_H +#define LLVM_TRANSFORMS_UTILS_ENTRYEXITINSTRUMENTER_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +struct EntryExitInstrumenterPass + : public PassInfoMixin<EntryExitInstrumenterPass> { + EntryExitInstrumenterPass(bool PostInlining) : PostInlining(PostInlining) {} + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + bool PostInlining; +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_ENTRYEXITINSTRUMENTER_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/Evaluator.h b/contrib/llvm/include/llvm/Transforms/Utils/Evaluator.h index 07f12f41b3bc..0e987b93177a 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/Evaluator.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/Evaluator.h @@ -1,4 +1,4 @@ -//===-- Evaluator.h - LLVM IR evaluator -------------------------*- C++ -*-===// +//===- Evaluator.h - LLVM IR evaluator --------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -18,9 +18,10 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constant.h" #include "llvm/IR/GlobalVariable.h" - +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include <cassert> #include <deque> #include <memory> @@ -114,6 +115,6 @@ private: const TargetLibraryInfo *TLI; }; -} +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_EVALUATOR_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/contrib/llvm/include/llvm/Transforms/Utils/FunctionComparator.h index b0f10eafaa95..7698a068717a 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/FunctionComparator.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/FunctionComparator.h @@ -15,10 +15,10 @@ #ifndef LLVM_TRANSFORMS_UTILS_FUNCTIONCOMPARATOR_H #define LLVM_TRANSFORMS_UTILS_FUNCTIONCOMPARATOR_H -#include "llvm/ADT/APFloat.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/IR/Function.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueMap.h" #include "llvm/Support/AtomicOrdering.h" @@ -28,7 +28,17 @@ namespace llvm { -class GetElementPtrInst; +class APFloat; +class APInt; +class BasicBlock; +class Constant; +class Function; +class GlobalValue; +class InlineAsm; +class Instruction; +class MDNode; +class Type; +class Value; /// GlobalNumberState assigns an integer to each global value in the program, /// which is used by the comparison routine to order references to globals. This @@ -43,14 +53,16 @@ class GetElementPtrInst; /// compare those, but this would not work for stripped bitcodes or for those /// few symbols without a name. class GlobalNumberState { - struct Config : ValueMapConfig<GlobalValue*> { + struct Config : ValueMapConfig<GlobalValue *> { enum { FollowRAUW = false }; }; + // Each GlobalValue is mapped to an identifier. The Config ensures when RAUW // occurs, the mapping does not change. Tracking changes is unnecessary, and // also problematic for weak symbols (which may be overwritten). - typedef ValueMap<GlobalValue *, uint64_t, Config> ValueNumberMap; + using ValueNumberMap = ValueMap<GlobalValue *, uint64_t, Config>; ValueNumberMap GlobalNumbers; + // The next unused serial number to assign to a global. uint64_t NextNumber = 0; @@ -66,6 +78,10 @@ public: return MapIter->second; } + void erase(GlobalValue *Global) { + GlobalNumbers.erase(Global); + } + void clear() { GlobalNumbers.clear(); } @@ -83,9 +99,10 @@ public: /// Test whether the two functions have equivalent behaviour. int compare(); + /// Hash a function. Equivalent functions will have the same hash, and unequal /// functions will have different hashes with high probability. - typedef uint64_t FunctionHash; + using FunctionHash = uint64_t; static FunctionHash functionHash(Function &); protected: diff --git a/contrib/llvm/include/llvm/Transforms/Utils/Local.h b/contrib/llvm/include/llvm/Transforms/Utils/Local.h index 30b27616cd98..01db88bc15c2 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/Local.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/Local.h @@ -1,4 +1,4 @@ -//===-- Local.h - Functions to perform local transformations ----*- C++ -*-===// +//===- Local.h - Functions to perform local transformations -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,39 +15,95 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOCAL_H #define LLVM_TRANSFORMS_UTILS_LOCAL_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/GetElementPtrTypeIterator.h" -#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include <cstdint> +#include <limits> namespace llvm { -class User; +class AllocaInst; +class AssumptionCache; class BasicBlock; -class Function; class BranchInst; -class Instruction; class CallInst; -class DbgDeclareInst; +class DbgInfoIntrinsic; class DbgValueInst; -class StoreInst; +class DIBuilder; +class Function; +class Instruction; +class LazyValueInfo; class LoadInst; -class Value; +class MDNode; class PHINode; -class AllocaInst; -class AssumptionCache; -class ConstantExpr; -class DataLayout; +class StoreInst; class TargetLibraryInfo; class TargetTransformInfo; -class DIBuilder; -class DominatorTree; -class LazyValueInfo; -template<typename T> class SmallVectorImpl; +/// A set of parameters used to control the transforms in the SimplifyCFG pass. +/// Options may change depending on the position in the optimization pipeline. +/// For example, canonical form that includes switches and branches may later be +/// replaced by lookup tables and selects. +struct SimplifyCFGOptions { + int BonusInstThreshold; + bool ForwardSwitchCondToPhi; + bool ConvertSwitchToLookupTable; + bool NeedCanonicalLoop; + bool SinkCommonInsts; + AssumptionCache *AC; + + SimplifyCFGOptions(unsigned BonusThreshold = 1, + bool ForwardSwitchCond = false, + bool SwitchToLookup = false, bool CanonicalLoops = true, + bool SinkCommon = false, + AssumptionCache *AssumpCache = nullptr) + : BonusInstThreshold(BonusThreshold), + ForwardSwitchCondToPhi(ForwardSwitchCond), + ConvertSwitchToLookupTable(SwitchToLookup), + NeedCanonicalLoop(CanonicalLoops), + SinkCommonInsts(SinkCommon), + AC(AssumpCache) {} + + // Support 'builder' pattern to set members by name at construction time. + SimplifyCFGOptions &bonusInstThreshold(int I) { + BonusInstThreshold = I; + return *this; + } + SimplifyCFGOptions &forwardSwitchCondToPhi(bool B) { + ForwardSwitchCondToPhi = B; + return *this; + } + SimplifyCFGOptions &convertSwitchToLookupTable(bool B) { + ConvertSwitchToLookupTable = B; + return *this; + } + SimplifyCFGOptions &needCanonicalLoops(bool B) { + NeedCanonicalLoop = B; + return *this; + } + SimplifyCFGOptions &sinkCommonInsts(bool B) { + SinkCommonInsts = B; + return *this; + } + SimplifyCFGOptions &setAssumptionCache(AssumptionCache *Cache) { + AC = Cache; + return *this; + } +}; //===----------------------------------------------------------------------===// // Local constant propagation. @@ -133,17 +189,15 @@ bool TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB); /// values, but instcombine orders them so it usually won't matter. bool EliminateDuplicatePHINodes(BasicBlock *BB); -/// This function is used to do simplification of a CFG. For -/// example, it adjusts branches to branches to eliminate the extra hop, it -/// eliminates unreachable basic blocks, and does other "peephole" optimization -/// of the CFG. It returns true if a modification was made, possibly deleting -/// the basic block that was pointed to. LoopHeaders is an optional input -/// parameter, providing the set of loop header that SimplifyCFG should not -/// eliminate. -bool SimplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, - unsigned BonusInstThreshold, AssumptionCache *AC = nullptr, - SmallPtrSetImpl<BasicBlock *> *LoopHeaders = nullptr, - bool LateSimplifyCFG = false); +/// This function is used to do simplification of a CFG. For example, it +/// adjusts branches to branches to eliminate the extra hop, it eliminates +/// unreachable basic blocks, and does other peephole optimization of the CFG. +/// It returns true if a modification was made, possibly deleting the basic +/// block that was pointed to. LoopHeaders is an optional input parameter +/// providing the set of loop headers that SimplifyCFG should not eliminate. +bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, + const SimplifyCFGOptions &Options = {}, + SmallPtrSetImpl<BasicBlock *> *LoopHeaders = nullptr); /// This function is used to flatten a CFG. For example, it uses parallel-and /// and parallel-or mode to collapse if-conditions and merge if-regions with @@ -185,10 +239,10 @@ unsigned getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign, const DominatorTree *DT = nullptr); /// Try to infer an alignment for the specified pointer. -static inline unsigned getKnownAlignment(Value *V, const DataLayout &DL, - const Instruction *CxtI = nullptr, - AssumptionCache *AC = nullptr, - const DominatorTree *DT = nullptr) { +inline unsigned getKnownAlignment(Value *V, const DataLayout &DL, + const Instruction *CxtI = nullptr, + AssumptionCache *AC = nullptr, + const DominatorTree *DT = nullptr) { return getOrEnforceKnownAlignment(V, 0, DL, CxtI, AC, DT); } @@ -210,7 +264,8 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, // Build a mask for high order bits. unsigned IntPtrWidth = IntPtrTy->getScalarType()->getIntegerBitWidth(); - uint64_t PtrSizeMask = ~0ULL >> (64 - IntPtrWidth); + uint64_t PtrSizeMask = + std::numeric_limits<uint64_t>::max() >> (64 - IntPtrWidth); gep_type_iterator GTI = gep_type_begin(GEP); for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); i != e; @@ -262,46 +317,50 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, /// /// Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value -/// that has an associated llvm.dbg.decl intrinsic. -void ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI, +/// that has an associated llvm.dbg.declare or llvm.dbg.addr intrinsic. +void ConvertDebugDeclareToDebugValue(DbgInfoIntrinsic *DII, StoreInst *SI, DIBuilder &Builder); /// Inserts a llvm.dbg.value intrinsic before a load of an alloca'd value -/// that has an associated llvm.dbg.decl intrinsic. -void ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI, +/// that has an associated llvm.dbg.declare or llvm.dbg.addr intrinsic. +void ConvertDebugDeclareToDebugValue(DbgInfoIntrinsic *DII, LoadInst *LI, DIBuilder &Builder); -/// Inserts a llvm.dbg.value intrinsic after a phi of an alloca'd value -/// that has an associated llvm.dbg.decl intrinsic. -void ConvertDebugDeclareToDebugValue(DbgDeclareInst *DDI, +/// Inserts a llvm.dbg.value intrinsic after a phi that has an associated +/// llvm.dbg.declare or llvm.dbg.addr intrinsic. +void ConvertDebugDeclareToDebugValue(DbgInfoIntrinsic *DII, PHINode *LI, DIBuilder &Builder); /// Lowers llvm.dbg.declare intrinsics into appropriate set of /// llvm.dbg.value intrinsics. bool LowerDbgDeclare(Function &F); -/// Finds the llvm.dbg.declare intrinsic corresponding to an alloca, if any. -DbgDeclareInst *FindAllocaDbgDeclare(Value *V); +/// Finds all intrinsics declaring local variables as living in the memory that +/// 'V' points to. This may include a mix of dbg.declare and +/// dbg.addr intrinsics. +TinyPtrVector<DbgInfoIntrinsic *> FindDbgAddrUses(Value *V); /// Finds the llvm.dbg.value intrinsics describing a value. void findDbgValues(SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V); -/// Replaces llvm.dbg.declare instruction when the address it describes -/// is replaced with a new value. If Deref is true, an additional DW_OP_deref is -/// prepended to the expression. If Offset is non-zero, a constant displacement -/// is added to the expression (after the optional Deref). Offset can be -/// negative. +/// Replaces llvm.dbg.declare instruction when the address it +/// describes is replaced with a new value. If Deref is true, an +/// additional DW_OP_deref is prepended to the expression. If Offset +/// is non-zero, a constant displacement is added to the expression +/// (between the optional Deref operations). Offset can be negative. bool replaceDbgDeclare(Value *Address, Value *NewAddress, Instruction *InsertBefore, DIBuilder &Builder, - bool Deref, int Offset); + bool DerefBefore, int Offset, bool DerefAfter); /// Replaces llvm.dbg.declare instruction when the alloca it describes -/// is replaced with a new value. If Deref is true, an additional DW_OP_deref is -/// prepended to the expression. If Offset is non-zero, a constant displacement -/// is added to the expression (after the optional Deref). Offset can be -/// negative. New llvm.dbg.declare is inserted immediately before AI. +/// is replaced with a new value. If Deref is true, an additional +/// DW_OP_deref is prepended to the expression. If Offset is non-zero, +/// a constant displacement is added to the expression (between the +/// optional Deref operations). Offset can be negative. The new +/// llvm.dbg.declare is inserted immediately before AI. bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, - DIBuilder &Builder, bool Deref, int Offset = 0); + DIBuilder &Builder, bool DerefBefore, + int Offset, bool DerefAfter); /// Replaces multiple llvm.dbg.value instructions when the alloca it describes /// is replaced with a new value. If Offset is non-zero, a constant displacement @@ -369,7 +428,6 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, const BasicBlock *BB); - /// Return true if the CallSite CS calls a gc leaf function. /// /// A leaf function is a function that does not safepoint the thread during its @@ -378,7 +436,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, /// /// Most passes can and should ignore this information, and it is only used /// during lowering by the GC infrastructure. -bool callsGCLeafFunction(ImmutableCallSite CS); +bool callsGCLeafFunction(ImmutableCallSite CS, const TargetLibraryInfo &TLI); /// Copy a nonnull metadata node to a new load instruction. /// @@ -397,7 +455,7 @@ void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N, // Intrinsic pattern matching // -/// Try and match a bswap or bitreverse idiom. +/// Try to match a bswap or bitreverse idiom. /// /// If an idiom is matched, an intrinsic call is inserted before \c I. Any added /// instructions are returned in \c InsertedInsts. They will all have been added @@ -431,6 +489,6 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, /// value? bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_LOCAL_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 94e20b83754e..750666136507 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -16,6 +16,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -305,10 +306,13 @@ public: /// induction, the induction descriptor \p D will contain the data describing /// this induction. If by some other means the caller has a better SCEV /// expression for \p Phi than the one returned by the ScalarEvolution - /// analysis, it can be passed through \p Expr. - static bool isInductionPHI(PHINode *Phi, const Loop* L, ScalarEvolution *SE, - InductionDescriptor &D, - const SCEV *Expr = nullptr); + /// analysis, it can be passed through \p Expr. If the def-use chain + /// associated with the phi includes casts (that we know we can ignore + /// under proper runtime checks), they are passed through \p CastsToIgnore. + static bool + isInductionPHI(PHINode *Phi, const Loop* L, ScalarEvolution *SE, + InductionDescriptor &D, const SCEV *Expr = nullptr, + SmallVectorImpl<Instruction *> *CastsToIgnore = nullptr); /// Returns true if \p Phi is a floating point induction in the loop \p L. /// If \p Phi is an induction, the induction descriptor \p D will contain @@ -330,15 +334,13 @@ public: /// not have the "fast-math" property. Such operation requires a relaxed FP /// mode. bool hasUnsafeAlgebra() { - return InductionBinOp && - !cast<FPMathOperator>(InductionBinOp)->hasUnsafeAlgebra(); + return InductionBinOp && !cast<FPMathOperator>(InductionBinOp)->isFast(); } /// Returns induction operator that does not have "fast-math" property /// and requires FP unsafe mode. Instruction *getUnsafeAlgebraInst() { - if (!InductionBinOp || - cast<FPMathOperator>(InductionBinOp)->hasUnsafeAlgebra()) + if (!InductionBinOp || cast<FPMathOperator>(InductionBinOp)->isFast()) return nullptr; return InductionBinOp; } @@ -349,10 +351,18 @@ public: Instruction::BinaryOpsEnd; } + /// Returns a reference to the type cast instructions in the induction + /// update chain, that are redundant when guarded with a runtime + /// SCEV overflow check. + const SmallVectorImpl<Instruction *> &getCastInsts() const { + return RedundantCasts; + } + private: /// Private constructor - used by \c isInductionPHI. InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, - BinaryOperator *InductionBinOp = nullptr); + BinaryOperator *InductionBinOp = nullptr, + SmallVectorImpl<Instruction *> *Casts = nullptr); /// Start value. TrackingVH<Value> StartValue; @@ -362,6 +372,9 @@ private: const SCEV *Step = nullptr; // Instruction that advances induction variable. BinaryOperator *InductionBinOp = nullptr; + // Instructions used for type-casts of the induction variable, + // that are redundant when guarded with a runtime SCEV overflow check. + SmallVector<Instruction *, 2> RedundantCasts; }; BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, @@ -423,8 +436,9 @@ bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, /// instructions of the loop and loop safety information as /// arguments. Diagnostics is emitted via \p ORE. It returns changed status. bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, - TargetLibraryInfo *, Loop *, AliasSetTracker *, - LoopSafetyInfo *, OptimizationRemarkEmitter *ORE); + TargetLibraryInfo *, TargetTransformInfo *, Loop *, + AliasSetTracker *, LoopSafetyInfo *, + OptimizationRemarkEmitter *ORE); /// \brief Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in depth @@ -438,21 +452,41 @@ bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, TargetLibraryInfo *, Loop *, AliasSetTracker *, LoopSafetyInfo *, OptimizationRemarkEmitter *ORE); +/// This function deletes dead loops. The caller of this function needs to +/// guarantee that the loop is infact dead. +/// The function requires a bunch or prerequisites to be present: +/// - The loop needs to be in LCSSA form +/// - The loop needs to have a Preheader +/// - A unique dedicated exit block must exist +/// +/// This also updates the relevant analysis information in \p DT, \p SE, and \p +/// LI if pointers to those are provided. +/// It also updates the loop PM if an updater struct is provided. + +void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE, + LoopInfo *LI); + /// \brief Try to promote memory values to scalars by sinking stores out of /// the loop and moving loads to before the loop. We do this by looping over /// the stores in the loop, looking for stores to Must pointers which are -/// loop invariant. It takes AliasSet, Loop exit blocks vector, loop exit blocks -/// insertion point vector, PredIteratorCache, LoopInfo, DominatorTree, Loop, -/// AliasSet information for all instructions of the loop and loop safety -/// information as arguments. Diagnostics is emitted via \p ORE. It returns -/// changed status. -bool promoteLoopAccessesToScalars(AliasSet &, SmallVectorImpl<BasicBlock *> &, +/// loop invariant. It takes a set of must-alias values, Loop exit blocks +/// vector, loop exit blocks insertion point vector, PredIteratorCache, +/// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions +/// of the loop and loop safety information as arguments. +/// Diagnostics is emitted via \p ORE. It returns changed status. +bool promoteLoopAccessesToScalars(const SmallSetVector<Value *, 8> &, + SmallVectorImpl<BasicBlock *> &, SmallVectorImpl<Instruction *> &, PredIteratorCache &, LoopInfo *, DominatorTree *, const TargetLibraryInfo *, Loop *, AliasSetTracker *, LoopSafetyInfo *, OptimizationRemarkEmitter *); +/// Does a BFS from a given node to all of its children inside a given loop. +/// The returned vector of nodes includes the starting point. +SmallVector<DomTreeNode *, 16> collectChildrenInLoop(DomTreeNode *N, + const Loop *CurLoop); + /// \brief Computes safety information for a loop /// checks loop body & header for the possibility of may throw /// exception, it takes LoopSafetyInfo and loop as argument. diff --git a/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h index 4554b5cbc644..2b7d0f67a324 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h @@ -25,12 +25,6 @@ class MemSetInst; class TargetTransformInfo; class Value; -/// Emit a loop implementing the semantics of llvm.memcpy with the equivalent -/// arguments at \p InsertBefore. -void createMemCpyLoop(Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, - Value *CopyLen, unsigned SrcAlign, unsigned DestAlign, - bool SrcIsVolatile, bool DstIsVolatile); - /// Emit a loop implementing the semantics of llvm.memcpy where the size is not /// a compile-time constant. Loop will be insterted at \p InsertBefore. void createMemCpyLoopUnknownSize(Instruction *InsertBefore, Value *SrcAddr, diff --git a/contrib/llvm/include/llvm/Transforms/Utils/Mem2Reg.h b/contrib/llvm/include/llvm/Transforms/Utils/Mem2Reg.h index 1fe186d6c3ad..407684338a3b 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/Mem2Reg.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/Mem2Reg.h @@ -15,14 +15,17 @@ #ifndef LLVM_TRANSFORMS_UTILS_MEM2REG_H #define LLVM_TRANSFORMS_UTILS_MEM2REG_H -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { + +class Function; + class PromotePass : public PassInfoMixin<PromotePass> { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_MEM2REG_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/contrib/llvm/include/llvm/Transforms/Utils/ModuleUtils.h index e9793fe4b666..4b9bc8293810 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -85,7 +85,8 @@ void filterDeadComdatFunctions( Module &M, SmallVectorImpl<Function *> &DeadComdatFunctions); /// \brief Produce a unique identifier for this module by taking the MD5 sum of -/// the names of the module's strong external symbols. +/// the names of the module's strong external symbols that are not comdat +/// members. /// /// This identifier is normally guaranteed to be unique, or the program would /// fail to link due to multiply defined symbols. diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h index 8cbcdf47156e..6cd9f1539b0b 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdater.h @@ -1,4 +1,4 @@ -//===-- SSAUpdater.h - Unstructured SSA Update Tool -------------*- C++ -*-===// +//===- SSAUpdater.h - Unstructured SSA Update Tool --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,6 +14,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_SSAUPDATER_H #define LLVM_TRANSFORMS_UTILS_SSAUPDATER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include <string> @@ -22,10 +23,9 @@ namespace llvm { class BasicBlock; class Instruction; class LoadInst; -template <typename T> class ArrayRef; +class PHINode; template <typename T> class SmallVectorImpl; template <typename T> class SSAUpdaterTraits; -class PHINode; class Type; class Use; class Value; @@ -42,7 +42,6 @@ class SSAUpdater { private: /// This keeps track of which value to use on a per-block basis. When we /// insert PHI nodes, we keep track of them here. - //typedef DenseMap<BasicBlock*, Value*> AvailableValsTy; void *AV = nullptr; /// ProtoType holds the type of the values being rewritten. @@ -53,12 +52,12 @@ private: /// If this is non-null, the SSAUpdater adds all PHI nodes that it creates to /// the vector. - SmallVectorImpl<PHINode*> *InsertedPHIs; + SmallVectorImpl<PHINode *> *InsertedPHIs; public: /// If InsertedPHIs is specified, it will be filled /// in with all PHI Nodes created by rewriting. - explicit SSAUpdater(SmallVectorImpl<PHINode*> *InsertedPHIs = nullptr); + explicit SSAUpdater(SmallVectorImpl<PHINode *> *InsertedPHIs = nullptr); SSAUpdater(const SSAUpdater &) = delete; SSAUpdater &operator=(const SSAUpdater &) = delete; ~SSAUpdater(); @@ -136,7 +135,7 @@ protected: SSAUpdater &SSA; public: - LoadAndStorePromoter(ArrayRef<const Instruction*> Insts, + LoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S, StringRef Name = StringRef()); virtual ~LoadAndStorePromoter() = default; @@ -145,32 +144,28 @@ public: /// Insts is a list of loads and stores to promote, and Name is the basename /// for the PHIs to insert. After this is complete, the loads and stores are /// removed from the code. - void run(const SmallVectorImpl<Instruction*> &Insts) const; + void run(const SmallVectorImpl<Instruction *> &Insts) const; /// \brief Return true if the specified instruction is in the Inst list. /// /// The Insts list is the one passed into the constructor. Clients should /// implement this with a more efficient version if possible. virtual bool isInstInList(Instruction *I, - const SmallVectorImpl<Instruction*> &Insts) const; + const SmallVectorImpl<Instruction *> &Insts) const; /// \brief This hook is invoked after all the stores are found and inserted as /// available values. - virtual void doExtraRewritesBeforeFinalDeletion() const { - } + virtual void doExtraRewritesBeforeFinalDeletion() const {} /// \brief Clients can choose to implement this to get notified right before /// a load is RAUW'd another value. - virtual void replaceLoadWithValue(LoadInst *LI, Value *V) const { - } + virtual void replaceLoadWithValue(LoadInst *LI, Value *V) const {} /// \brief Called before each instruction is deleted. - virtual void instructionDeleted(Instruction *I) const { - } + virtual void instructionDeleted(Instruction *I) const {} /// \brief Called to update debug info associated with the instruction. - virtual void updateDebugInfo(Instruction *I) const { - } + virtual void updateDebugInfo(Instruction *I) const {} }; } // end namespace llvm diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h index 2dd205d8b2af..b1611d49a456 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h @@ -1,4 +1,4 @@ -//===-- SSAUpdaterImpl.h - SSA Updater Implementation -----------*- C++ -*-===// +//===- SSAUpdaterImpl.h - SSA Updater Implementation ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -17,17 +17,14 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/ValueHandle.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "ssaupdater" namespace llvm { -class CastInst; -class PHINode; template<typename T> class SSAUpdaterTraits; template<typename UpdaterT> @@ -35,51 +32,67 @@ class SSAUpdaterImpl { private: UpdaterT *Updater; - typedef SSAUpdaterTraits<UpdaterT> Traits; - typedef typename Traits::BlkT BlkT; - typedef typename Traits::ValT ValT; - typedef typename Traits::PhiT PhiT; + using Traits = SSAUpdaterTraits<UpdaterT>; + using BlkT = typename Traits::BlkT; + using ValT = typename Traits::ValT; + using PhiT = typename Traits::PhiT; /// BBInfo - Per-basic block information used internally by SSAUpdaterImpl. /// The predecessors of each block are cached here since pred_iterator is /// slow and we need to iterate over the blocks at least a few times. class BBInfo { public: - BlkT *BB; // Back-pointer to the corresponding block. - ValT AvailableVal; // Value to use in this block. - BBInfo *DefBB; // Block that defines the available value. - int BlkNum; // Postorder number. - BBInfo *IDom; // Immediate dominator. - unsigned NumPreds; // Number of predecessor blocks. - BBInfo **Preds; // Array[NumPreds] of predecessor blocks. - PhiT *PHITag; // Marker for existing PHIs that match. + // Back-pointer to the corresponding block. + BlkT *BB; + + // Value to use in this block. + ValT AvailableVal; + + // Block that defines the available value. + BBInfo *DefBB; + + // Postorder number. + int BlkNum = 0; + + // Immediate dominator. + BBInfo *IDom = nullptr; + + // Number of predecessor blocks. + unsigned NumPreds = 0; + + // Array[NumPreds] of predecessor blocks. + BBInfo **Preds = nullptr; + + // Marker for existing PHIs that match. + PhiT *PHITag = nullptr; BBInfo(BlkT *ThisBB, ValT V) - : BB(ThisBB), AvailableVal(V), DefBB(V ? this : nullptr), BlkNum(0), - IDom(nullptr), NumPreds(0), Preds(nullptr), PHITag(nullptr) {} + : BB(ThisBB), AvailableVal(V), DefBB(V ? this : nullptr) {} }; - typedef DenseMap<BlkT*, ValT> AvailableValsTy; + using AvailableValsTy = DenseMap<BlkT *, ValT>; + AvailableValsTy *AvailableVals; - SmallVectorImpl<PhiT*> *InsertedPHIs; + SmallVectorImpl<PhiT *> *InsertedPHIs; + + using BlockListTy = SmallVectorImpl<BBInfo *>; + using BBMapTy = DenseMap<BlkT *, BBInfo *>; - typedef SmallVectorImpl<BBInfo*> BlockListTy; - typedef DenseMap<BlkT*, BBInfo*> BBMapTy; BBMapTy BBMap; BumpPtrAllocator Allocator; public: explicit SSAUpdaterImpl(UpdaterT *U, AvailableValsTy *A, - SmallVectorImpl<PhiT*> *Ins) : - Updater(U), AvailableVals(A), InsertedPHIs(Ins) { } + SmallVectorImpl<PhiT *> *Ins) : + Updater(U), AvailableVals(A), InsertedPHIs(Ins) {} /// GetValue - Check to see if AvailableVals has an entry for the specified /// BB and if so, return it. If not, construct SSA form by first /// calculating the required placement of PHIs and then inserting new PHIs /// where needed. ValT GetValue(BlkT *BB) { - SmallVector<BBInfo*, 100> BlockList; + SmallVector<BBInfo *, 100> BlockList; BBInfo *PseudoEntry = BuildBlockList(BB, &BlockList); // Special case: bail out if BB is unreachable. @@ -101,8 +114,8 @@ public: /// Create BBInfo structures for the blocks and append them to the block /// list. BBInfo *BuildBlockList(BlkT *BB, BlockListTy *BlockList) { - SmallVector<BBInfo*, 10> RootList; - SmallVector<BBInfo*, 64> WorkList; + SmallVector<BBInfo *, 10> RootList; + SmallVector<BBInfo *, 64> WorkList; BBInfo *Info = new (Allocator) BBInfo(BB, 0); BBMap[BB] = Info; @@ -111,7 +124,7 @@ public: // Search backward from BB, creating BBInfos along the way and stopping // when reaching blocks that define the value. Record those defining // blocks on the RootList. - SmallVector<BlkT*, 10> Preds; + SmallVector<BlkT *, 10> Preds; while (!WorkList.empty()) { Info = WorkList.pop_back_val(); Preds.clear(); @@ -395,7 +408,7 @@ public: /// CheckIfPHIMatches - Check if a PHI node matches the placement and values /// in the BBMap. bool CheckIfPHIMatches(PhiT *PHI) { - SmallVector<PhiT*, 20> WorkList; + SmallVector<PhiT *, 20> WorkList; WorkList.push_back(PHI); // Mark that the block containing this PHI has been visited. @@ -453,7 +466,7 @@ public: } }; -} // end llvm namespace +} // end namespace llvm #undef DEBUG_TYPE // "ssaupdater" diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h index 8d50aeb10d6e..a1dfed29a22d 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -26,6 +26,7 @@ class Loop; class LoopInfo; class PHINode; class ScalarEvolution; +class SCEVExpander; /// Interface for visiting interesting IV users that are recognized but not /// simplified by this utility. @@ -47,7 +48,7 @@ public: /// by using ScalarEvolution to analyze the IV's recurrence. bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT, LoopInfo *LI, SmallVectorImpl<WeakTrackingVH> &Dead, - IVVisitor *V = nullptr); + SCEVExpander &Rewriter, IVVisitor *V = nullptr); /// SimplifyLoopIVs - Simplify users of induction variables within this /// loop. This does not actually change or add IVs. diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h index 6aba9b2298b1..73a62f59203b 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -28,6 +28,7 @@ class Instruction; class TargetLibraryInfo; class BasicBlock; class Function; +class OptimizationRemarkEmitter; /// \brief This class implements simplifications for calls to fortified library /// functions (__st*cpy_chk, __memcpy_chk, __memmove_chk, __memset_chk), to, @@ -73,6 +74,7 @@ private: FortifiedLibCallSimplifier FortifiedSimplifier; const DataLayout &DL; const TargetLibraryInfo *TLI; + OptimizationRemarkEmitter &ORE; bool UnsafeFPShrink; function_ref<void(Instruction *, Value *)> Replacer; @@ -87,6 +89,7 @@ private: public: LibCallSimplifier(const DataLayout &DL, const TargetLibraryInfo *TLI, + OptimizationRemarkEmitter &ORE, function_ref<void(Instruction *, Value *)> Replacer = &replaceAllUsesWithDefault); @@ -126,14 +129,19 @@ private: Value *optimizeStringMemoryLibCall(CallInst *CI, IRBuilder<> &B); // Math Library Optimizations + Value *optimizeCAbs(CallInst *CI, IRBuilder<> &B); Value *optimizeCos(CallInst *CI, IRBuilder<> &B); Value *optimizePow(CallInst *CI, IRBuilder<> &B); + Value *replacePowWithSqrt(CallInst *Pow, IRBuilder<> &B); Value *optimizeExp2(CallInst *CI, IRBuilder<> &B); Value *optimizeFMinFMax(CallInst *CI, IRBuilder<> &B); Value *optimizeLog(CallInst *CI, IRBuilder<> &B); Value *optimizeSqrt(CallInst *CI, IRBuilder<> &B); Value *optimizeSinCosPi(CallInst *CI, IRBuilder<> &B); Value *optimizeTan(CallInst *CI, IRBuilder<> &B); + // Wrapper for all floating point library call optimizations + Value *optimizeFloatingPointLibCall(CallInst *CI, LibFunc Func, + IRBuilder<> &B); // Integer Library Call Optimizations Value *optimizeFFS(CallInst *CI, IRBuilder<> &B); diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SplitModule.h b/contrib/llvm/include/llvm/Transforms/Utils/SplitModule.h index b7a3bcf4f86a..d2c31f2701ac 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SplitModule.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SplitModule.h @@ -22,7 +22,6 @@ namespace llvm { class Module; -class StringRef; /// Splits the module M into N linkable partitions. The function ModuleCallback /// is called N times passing each individual partition as the MPart argument. @@ -39,6 +38,6 @@ void SplitModule( function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback, bool PreserveLocals = false); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_SPLITMODULE_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h b/contrib/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h index 93658989fba5..e0caf7741ff3 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/SymbolRewriter.h @@ -1,4 +1,4 @@ -//===-- SymbolRewriter.h - Symbol Rewriting Pass ----------------*- C++ -*-===// +//===- SymbolRewriter.h - Symbol Rewriting Pass -----------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -33,7 +33,6 @@ #ifndef LLVM_TRANSFORMS_UTILS_SYMBOLREWRITER_H #define LLVM_TRANSFORMS_UTILS_SYMBOLREWRITER_H -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include <list> #include <memory> @@ -42,6 +41,8 @@ namespace llvm { class MemoryBuffer; +class Module; +class ModulePass; namespace yaml { @@ -89,7 +90,7 @@ private: const Type Kind; }; -typedef std::list<std::unique_ptr<RewriteDescriptor>> RewriteDescriptorList; +using RewriteDescriptorList = std::list<std::unique_ptr<RewriteDescriptor>>; class RewriteMapParser { public: @@ -120,6 +121,7 @@ ModulePass *createRewriteSymbolsPass(SymbolRewriter::RewriteDescriptorList &); class RewriteSymbolPass : public PassInfoMixin<RewriteSymbolPass> { public: RewriteSymbolPass() { loadAndParseMapFiles(); } + RewriteSymbolPass(SymbolRewriter::RewriteDescriptorList &DL) { Descriptors.splice(Descriptors.begin(), DL); } diff --git a/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h index a3115ad16914..12aa3bc6e770 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/UnrollLoop.h @@ -16,40 +16,57 @@ #ifndef LLVM_TRANSFORMS_UTILS_UNROLLLOOP_H #define LLVM_TRANSFORMS_UTILS_UNROLLLOOP_H -// Needed because we can't forward-declare the nested struct -// TargetTransformInfo::UnrollingPreferences +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetTransformInfo.h" namespace llvm { -class StringRef; class AssumptionCache; +class BasicBlock; class DominatorTree; class Loop; class LoopInfo; -class LPPassManager; class MDNode; -class Pass; class OptimizationRemarkEmitter; class ScalarEvolution; -typedef SmallDenseMap<const Loop *, Loop *, 4> NewLoopsMap; +using NewLoopsMap = SmallDenseMap<const Loop *, Loop *, 4>; const Loop* addClonedBlockToLoopInfo(BasicBlock *OriginalBB, BasicBlock *ClonedBB, LoopInfo *LI, NewLoopsMap &NewLoops); -bool UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool Force, - bool AllowRuntime, bool AllowExpensiveTripCount, - bool PreserveCondBr, bool PreserveOnlyFirst, - unsigned TripMultiple, unsigned PeelCount, LoopInfo *LI, - ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, - OptimizationRemarkEmitter *ORE, bool PreserveLCSSA); +/// Represents the result of a \c UnrollLoop invocation. +enum class LoopUnrollResult { + /// The loop was not modified. + Unmodified, + + /// The loop was partially unrolled -- we still have a loop, but with a + /// smaller trip count. We may also have emitted epilogue loop if the loop + /// had a non-constant trip count. + PartiallyUnrolled, + + /// The loop was fully unrolled into straight-line code. We no longer have + /// any back-edges. + FullyUnrolled +}; + +LoopUnrollResult UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, + bool Force, bool AllowRuntime, + bool AllowExpensiveTripCount, bool PreserveCondBr, + bool PreserveOnlyFirst, unsigned TripMultiple, + unsigned PeelCount, bool UnrollRemainder, + LoopInfo *LI, ScalarEvolution *SE, + DominatorTree *DT, AssumptionCache *AC, + OptimizationRemarkEmitter *ORE, bool PreserveLCSSA); bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, - bool UseEpilogRemainder, LoopInfo *LI, + bool UseEpilogRemainder, bool UnrollRemainder, + LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, bool PreserveLCSSA); void computePeelCount(Loop *L, unsigned LoopSize, @@ -60,6 +77,7 @@ bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA); MDNode *GetUnrollMetadata(MDNode *LoopID, StringRef Name); -} -#endif +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_UNROLLLOOP_H diff --git a/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h index 45ef8246dcd1..4ecb23ea1951 100644 --- a/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h +++ b/contrib/llvm/include/llvm/Transforms/Utils/ValueMapper.h @@ -21,9 +21,17 @@ namespace llvm { -class Value; +class Constant; +class Function; +class GlobalAlias; +class GlobalVariable; class Instruction; -typedef ValueMap<const Value *, WeakTrackingVH> ValueToValueMapTy; +class MDNode; +class Metadata; +class Type; +class Value; + +using ValueToValueMapTy = ValueMap<const Value *, WeakTrackingVH>; /// This is a class that can be implemented by clients to remap types when /// cloning constants and instructions. @@ -44,10 +52,10 @@ class ValueMaterializer { virtual void anchor(); // Out of line method. protected: - ~ValueMaterializer() = default; ValueMaterializer() = default; ValueMaterializer(const ValueMaterializer &) = default; ValueMaterializer &operator=(const ValueMaterializer &) = default; + ~ValueMaterializer() = default; public: /// This method can be implemented to generate a mapped Value on demand. For @@ -91,7 +99,7 @@ enum RemapFlags { RF_NullMapMissingGlobalValues = 8, }; -static inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { +inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { return RemapFlags(unsigned(LHS) | unsigned(RHS)); } diff --git a/contrib/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/contrib/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h index 57d10c4c7473..32b56d372ea1 100644 --- a/contrib/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/contrib/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -1,4 +1,4 @@ -//===---- LoopVectorize.h ---------------------------------------*- C++ -*-===// +//===- LoopVectorize.h ------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -49,27 +49,29 @@ #ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H -#include "llvm/ADT/MapVector.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/BasicAliasAnalysis.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/DemandedBits.h" -#include "llvm/Analysis/LoopAccessAnalysis.h" -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/OptimizationDiagnosticInfo.h" -#include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Scalar/LoopPassManager.h" #include <functional> namespace llvm { +class AssumptionCache; +class BlockFrequencyInfo; +class DemandedBits; +class DominatorTree; +class Function; +class Loop; +class LoopAccessInfo; +class LoopInfo; +class OptimizationRemarkEmitter; +class ScalarEvolution; +class TargetLibraryInfo; +class TargetTransformInfo; + /// The LoopVectorize Pass. struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { bool DisableUnrolling = false; + /// If true, consider all loops for vectorization. /// If false, only loops that explicitly request vectorization are /// considered. @@ -99,6 +101,7 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> { bool processLoop(Loop *L); }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H diff --git a/contrib/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/contrib/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 6f258191e89e..25f264c4722c 100644 --- a/contrib/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/contrib/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -1,4 +1,4 @@ -//===---- SLPVectorizer.h ---------------------------------------*- C++ -*-===// +//===- SLPVectorizer.h ------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -19,30 +19,48 @@ #ifndef LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H #define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/DemandedBits.h" -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/OptimizationDiagnosticInfo.h" -#include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ValueHandle.h" namespace llvm { +class AssumptionCache; +class BasicBlock; +class CmpInst; +class DataLayout; +class DemandedBits; +class DominatorTree; +class Function; +class InsertElementInst; +class InsertValueInst; +class Instruction; +class LoopInfo; +class OptimizationRemarkEmitter; +class PHINode; +class ScalarEvolution; +class StoreInst; +class TargetLibraryInfo; +class TargetTransformInfo; +class Value; + /// A private "module" namespace for types and utilities used by this pass. /// These are implementation details and should not be used by clients. namespace slpvectorizer { + class BoUpSLP; -} + +} // end namespace slpvectorizer struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> { - typedef SmallVector<StoreInst *, 8> StoreList; - typedef MapVector<Value *, StoreList> StoreListMap; - typedef SmallVector<WeakTrackingVH, 8> WeakTrackingVHList; - typedef MapVector<Value *, WeakTrackingVHList> WeakTrackingVHListMap; + using StoreList = SmallVector<StoreInst *, 8>; + using StoreListMap = MapVector<Value *, StoreList>; + using WeakTrackingVHList = SmallVector<WeakTrackingVH, 8>; + using WeakTrackingVHListMap = MapVector<Value *, WeakTrackingVHList>; ScalarEvolution *SE = nullptr; TargetTransformInfo *TTI = nullptr; @@ -78,14 +96,16 @@ private: /// \brief Try to vectorize a list of operands. /// \@param BuildVector A list of users to ignore for the purpose of - /// scheduling and that don't need extracting. + /// scheduling and cost estimation when NeedExtraction + /// is false. /// \returns true if a value was vectorized. bool tryToVectorizeList(ArrayRef<Value *> VL, slpvectorizer::BoUpSLP &R, ArrayRef<Value *> BuildVector = None, - bool AllowReorder = false); + bool AllowReorder = false, + bool NeedExtraction = false); - /// \brief Try to vectorize a chain that may start at the operands of \p V. - bool tryToVectorize(BinaryOperator *V, slpvectorizer::BoUpSLP &R); + /// \brief Try to vectorize a chain that may start at the operands of \p I. + bool tryToVectorize(Instruction *I, slpvectorizer::BoUpSLP &R); /// \brief Vectorize the store instructions collected in Stores. bool vectorizeStoreChains(slpvectorizer::BoUpSLP &R); @@ -100,6 +120,22 @@ private: slpvectorizer::BoUpSLP &R, TargetTransformInfo *TTI); + /// Try to vectorize trees that start at insertvalue instructions. + bool vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, + slpvectorizer::BoUpSLP &R); + + /// Try to vectorize trees that start at insertelement instructions. + bool vectorizeInsertElementInst(InsertElementInst *IEI, BasicBlock *BB, + slpvectorizer::BoUpSLP &R); + + /// Try to vectorize trees that start at compare instructions. + bool vectorizeCmpInst(CmpInst *CI, BasicBlock *BB, slpvectorizer::BoUpSLP &R); + + /// Tries to vectorize constructs started from CmpInst, InsertValueInst or + /// InsertElementInst instructions. + bool vectorizeSimpleInstructions(SmallVectorImpl<WeakVH> &Instructions, + BasicBlock *BB, slpvectorizer::BoUpSLP &R); + /// \brief Scan the basic block and look for patterns that are likely to start /// a vectorization chain. bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R); @@ -115,6 +151,7 @@ private: /// The getelementptr instructions in a basic block organized by base pointer. WeakTrackingVHListMap GEPs; }; -} + +} // end namespace llvm #endif // LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H diff --git a/contrib/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h b/contrib/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h new file mode 100644 index 000000000000..302d3705887b --- /dev/null +++ b/contrib/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h @@ -0,0 +1,66 @@ +//===-- WindowsManifestMerger.h ---------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This file provides a utility for merging Microsoft .manifest files. These +// files are xml documents which contain meta-information about applications, +// such as whether or not admin access is required, system compatibility, +// versions, etc. Part of the linking process of an executable may require +// merging several of these .manifest files using a tree-merge following +// specific rules. Unfortunately, these rules are not documented well +// anywhere. However, a careful investigation of the behavior of the original +// Microsoft Manifest Tool (mt.exe) revealed the rules of this merge. As the +// saying goes, code is the best documentation, so please look below if you are +// interested in the exact merging requirements. +// +// Ref: +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa374191(v=vs.85).aspx +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_MANIFEST_MERGER_H +#define LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_MANIFEST_MERGER_H + +#include "llvm/Support/Error.h" + +namespace llvm { + +class MemoryBuffer; + +namespace windows_manifest { + +bool isAvailable(); + +class WindowsManifestError : public ErrorInfo<WindowsManifestError, ECError> { +public: + static char ID; + WindowsManifestError(const Twine &Msg); + void log(raw_ostream &OS) const override; + +private: + std::string Msg; +}; + +class WindowsManifestMerger { +public: + WindowsManifestMerger(); + ~WindowsManifestMerger(); + Error merge(const MemoryBuffer &Manifest); + + // Returns vector containing merged xml manifest, or uninitialized vector for + // empty manifest. + std::unique_ptr<MemoryBuffer> getMergedManifest(); + +private: + class WindowsManifestMergerImpl; + std::unique_ptr<WindowsManifestMergerImpl> Impl; +}; + +} // namespace windows_manifest +} // namespace llvm +#endif diff --git a/contrib/llvm/include/llvm/WindowsResource/ResourceProcessor.h b/contrib/llvm/include/llvm/WindowsResource/ResourceProcessor.h new file mode 100644 index 000000000000..4ca0a4b05bd0 --- /dev/null +++ b/contrib/llvm/include/llvm/WindowsResource/ResourceProcessor.h @@ -0,0 +1,51 @@ +//===-- ResourceProcessor.h -------------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_RESOURCE_PROCESSOR_H +#define LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_RESOURCE_PROCESSOR_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" + +#include <memory> +#include <vector> + + +namespace llvm { + +class WindowsResourceProcessor { +public: + using PathType = SmallVector<char, 64>; + + WindowsResourceProcessor() {} + + void addDefine(StringRef Key, StringRef Value = StringRef()) { + PreprocessorDefines.emplace_back(Key, Value); + } + void addInclude(const PathType &IncludePath) { + IncludeList.push_back(IncludePath); + } + void setVerbose(bool Verbose) { IsVerbose = Verbose; } + void setNullAtEnd(bool NullAtEnd) { AppendNull = NullAtEnd; } + + Error process(StringRef InputData, + std::unique_ptr<raw_fd_ostream> OutputStream); + +private: + StringRef InputData; + std::vector<PathType> IncludeList; + std::vector<std::pair<StringRef, StringRef>> PreprocessorDefines; + bool IsVerbose, AppendNull; +}; + +} + +#endif diff --git a/contrib/llvm/include/llvm/WindowsResource/ResourceScriptToken.h b/contrib/llvm/include/llvm/WindowsResource/ResourceScriptToken.h new file mode 100644 index 000000000000..494ae3222a4b --- /dev/null +++ b/contrib/llvm/include/llvm/WindowsResource/ResourceScriptToken.h @@ -0,0 +1,59 @@ +//===-- ResourceScriptToken.h -----------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This declares the .rc script tokens. +// The list of available tokens is located at ResourceScriptTokenList.h. +// +// Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa380599(v=vs.85).aspx +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_RESOURCE_SCRIPTTOKEN_H +#define LLVM_INCLUDE_LLVM_SUPPORT_WINDOWS_RESOURCE_SCRIPTTOKEN_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { + +// A definition of a single resource script token. Each token has its kind +// (declared in ResourceScriptTokenList) and holds a value - a reference +// representation of the token. +// RCToken does not claim ownership on its value. A memory buffer containing +// the token value should be stored in a safe place and cannot be freed +// nor reallocated. +class RCToken { +public: + enum class Kind { +#define TOKEN(Name) Name, +#define SHORT_TOKEN(Name, Ch) Name, +#include "ResourceScriptTokenList.h" +#undef TOKEN +#undef SHORT_TOKEN + }; + + RCToken(RCToken::Kind RCTokenKind, StringRef Value); + + // Get an integer value of the integer token. + uint32_t intValue() const; + bool isLongInt() const; + + StringRef value() const; + Kind kind() const; + + // Check if a token describes a binary operator. + bool isBinaryOp() const; + +private: + Kind TokenKind; + StringRef TokenValue; +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm/include/llvm/WindowsResource/ResourceScriptTokenList.h b/contrib/llvm/include/llvm/WindowsResource/ResourceScriptTokenList.h new file mode 100644 index 000000000000..0beed117c3e7 --- /dev/null +++ b/contrib/llvm/include/llvm/WindowsResource/ResourceScriptTokenList.h @@ -0,0 +1,35 @@ +//===-- ResourceScriptTokenList.h -------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// +// This is a part of llvm-rc tokens header. It lists all the possible tokens +// that might occur in a correct .rc script. +// +//===---------------------------------------------------------------------===// + + +// Long tokens. They might consist of more than one character. +TOKEN(Invalid) // Invalid token. Should not occur in a valid script. +TOKEN(Int) // Integer (decimal, octal or hexadecimal). +TOKEN(String) // String value. +TOKEN(Identifier) // Script identifier (resource name or type). + +// Short tokens. They usually consist of exactly one character. +// The definitions are of the form SHORT_TOKEN(TokenName, TokenChar). +// TokenChar is the one-character token representation occuring in the correct +// .rc scripts. +SHORT_TOKEN(BlockBegin, '{') // Start of the script block; can also be BEGIN. +SHORT_TOKEN(BlockEnd, '}') // End of the block; can also be END. +SHORT_TOKEN(Comma, ',') // Comma - resource arguments separator. +SHORT_TOKEN(Plus, '+') // Addition operator. +SHORT_TOKEN(Minus, '-') // Subtraction operator. +SHORT_TOKEN(Pipe, '|') // Bitwise-OR operator. +SHORT_TOKEN(Amp, '&') // Bitwise-AND operator. +SHORT_TOKEN(Tilde, '~') // Bitwise-NOT operator. +SHORT_TOKEN(LeftParen, '(') // Left parenthesis in the script expressions. +SHORT_TOKEN(RightParen, ')') // Right parenthesis. diff --git a/contrib/llvm/include/llvm/XRay/InstrumentationMap.h b/contrib/llvm/include/llvm/XRay/InstrumentationMap.h index 0342da0a2f0f..42bfca36a20b 100644 --- a/contrib/llvm/include/llvm/XRay/InstrumentationMap.h +++ b/contrib/llvm/include/llvm/XRay/InstrumentationMap.h @@ -38,7 +38,7 @@ Expected<InstrumentationMap> loadInstrumentationMap(StringRef Filename); struct SledEntry { /// Each entry here represents the kinds of supported instrumentation map /// entries. - enum class FunctionKinds { ENTRY, EXIT, TAIL }; + enum class FunctionKinds { ENTRY, EXIT, TAIL, LOG_ARGS_ENTER, CUSTOM_EVENT }; /// The address of the sled. uint64_t Address; @@ -106,6 +106,10 @@ template <> struct ScalarEnumerationTraits<xray::SledEntry::FunctionKinds> { IO.enumCase(Kind, "function-enter", xray::SledEntry::FunctionKinds::ENTRY); IO.enumCase(Kind, "function-exit", xray::SledEntry::FunctionKinds::EXIT); IO.enumCase(Kind, "tail-exit", xray::SledEntry::FunctionKinds::TAIL); + IO.enumCase(Kind, "log-args-enter", + xray::SledEntry::FunctionKinds::LOG_ARGS_ENTER); + IO.enumCase(Kind, "custom-event", + xray::SledEntry::FunctionKinds::CUSTOM_EVENT); } }; diff --git a/contrib/llvm/include/llvm/XRay/XRayRecord.h b/contrib/llvm/include/llvm/XRay/XRayRecord.h index 68c91a40fed1..5c5e9f436f4a 100644 --- a/contrib/llvm/include/llvm/XRay/XRayRecord.h +++ b/contrib/llvm/include/llvm/XRay/XRayRecord.h @@ -16,6 +16,7 @@ #define LLVM_XRAY_XRAY_RECORD_H #include <cstdint> +#include <vector> namespace llvm { namespace xray { @@ -53,7 +54,7 @@ struct XRayFileHeader { /// This may or may not correspond to actual record types in the raw trace (as /// the loader implementation may synthesize this information in the process of /// of loading). -enum class RecordTypes { ENTER, EXIT }; +enum class RecordTypes { ENTER, EXIT, TAIL_EXIT, ENTER_ARG }; struct XRayRecord { /// The type of record. @@ -73,6 +74,9 @@ struct XRayRecord { /// The thread ID for the currently running thread. uint32_t TId; + + /// The function call arguments. + std::vector<uint64_t> CallArgs; }; } // namespace xray diff --git a/contrib/llvm/include/llvm/XRay/YAMLXRayRecord.h b/contrib/llvm/include/llvm/XRay/YAMLXRayRecord.h index 7e1a4112818e..b436aefb1e8f 100644 --- a/contrib/llvm/include/llvm/XRay/YAMLXRayRecord.h +++ b/contrib/llvm/include/llvm/XRay/YAMLXRayRecord.h @@ -37,6 +37,7 @@ struct YAMLXRayRecord { std::string Function; uint64_t TSC; uint32_t TId; + std::vector<uint64_t> CallArgs; }; struct YAMLXRayTrace { @@ -54,6 +55,8 @@ template <> struct ScalarEnumerationTraits<xray::RecordTypes> { static void enumeration(IO &IO, xray::RecordTypes &Type) { IO.enumCase(Type, "function-enter", xray::RecordTypes::ENTER); IO.enumCase(Type, "function-exit", xray::RecordTypes::EXIT); + IO.enumCase(Type, "function-tail-exit", xray::RecordTypes::TAIL_EXIT); + IO.enumCase(Type, "function-enter-arg", xray::RecordTypes::ENTER_ARG); } }; @@ -73,6 +76,7 @@ template <> struct MappingTraits<xray::YAMLXRayRecord> { IO.mapRequired("type", Record.RecordType); IO.mapRequired("func-id", Record.FuncId); IO.mapOptional("function", Record.Function); + IO.mapOptional("args", Record.CallArgs); IO.mapRequired("cpu", Record.CPU); IO.mapRequired("thread", Record.TId); IO.mapRequired("kind", Record.Type); diff --git a/contrib/llvm/include/llvm/module.modulemap b/contrib/llvm/include/llvm/module.modulemap index 766198bbc5de..382942be64a1 100644 --- a/contrib/llvm/include/llvm/module.modulemap +++ b/contrib/llvm/include/llvm/module.modulemap @@ -23,18 +23,17 @@ module LLVM_Backend { exclude header "CodeGen/CommandFlags.h" exclude header "CodeGen/LinkAllAsmWriterComponents.h" exclude header "CodeGen/LinkAllCodegenComponents.h" - exclude header "CodeGen/GlobalISel/InstructionSelectorImpl.h" // These are intended for (repeated) textual inclusion. + textual header "CodeGen/CommandFlags.def" textual header "CodeGen/DIEValue.def" + textual header "CodeGen/RuntimeLibcalls.def" + textual header "CodeGen/TargetOpcodes.def" } module Target { umbrella "Target" module * { export * } - - // This is intended for (repeated) textual inclusion. - textual header "Target/TargetOpcodes.def" } } @@ -48,6 +47,7 @@ module LLVM_BinaryFormat { textual header "BinaryFormat/ELFRelocs/AArch64.def" textual header "BinaryFormat/ELFRelocs/AMDGPU.def" textual header "BinaryFormat/ELFRelocs/ARM.def" + textual header "BinaryFormat/ELFRelocs/ARC.def" textual header "BinaryFormat/ELFRelocs/AVR.def" textual header "BinaryFormat/ELFRelocs/BPF.def" textual header "BinaryFormat/ELFRelocs/Hexagon.def" @@ -93,11 +93,13 @@ module LLVM_DebugInfo_PDB { exclude header "DebugInfo/PDB/DIA/DIAEnumLineNumbers.h" exclude header "DebugInfo/PDB/DIA/DIAEnumSourceFiles.h" exclude header "DebugInfo/PDB/DIA/DIAEnumSymbols.h" + exclude header "DebugInfo/PDB/DIA/DIAEnumTables.h" exclude header "DebugInfo/PDB/DIA/DIALineNumber.h" exclude header "DebugInfo/PDB/DIA/DIARawSymbol.h" exclude header "DebugInfo/PDB/DIA/DIASession.h" exclude header "DebugInfo/PDB/DIA/DIASourceFile.h" exclude header "DebugInfo/PDB/DIA/DIASupport.h" + exclude header "DebugInfo/PDB/DIA/DIATable.h" } module LLVM_DebugInfo_PDB_DIA { @@ -121,6 +123,7 @@ module LLVM_DebugInfo_CodeView { module * { export * } // These are intended for (repeated) textual inclusion. + textual header "DebugInfo/CodeView/CodeViewRegisters.def" textual header "DebugInfo/CodeView/CodeViewTypes.def" textual header "DebugInfo/CodeView/CodeViewSymbols.def" } @@ -147,6 +150,7 @@ module LLVM_ExecutionEngine { exclude header "ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" exclude header "ExecutionEngine/Orc/OrcRemoteTargetClient.h" exclude header "ExecutionEngine/Orc/OrcRemoteTargetServer.h" + exclude header "ExecutionEngine/Orc/RemoteObjectLayer.h" } module LLVM_Pass { @@ -222,17 +226,20 @@ module LLVM_LTO { requires cplusplus umbrella "LTO" module * { export * } } module LLVM_MC { requires cplusplus - // FIXME: Mislayered? - module Support_TargetRegistry { - header "Support/TargetRegistry.h" - export * - } - umbrella "MC" module * { export * } - // Exclude this; it's fundamentally non-modular. - exclude header "MC/MCTargetOptionsCommandFlags.h" + textual header "MC/MCTargetOptionsCommandFlags.def" +} + +// Used by llvm-tblgen +module LLVM_MC_TableGen { + requires cplusplus + module MC_LaneBitmask { header "MC/LaneBitmask.h" export * } + module MC_FixedLenDisassembler { header "MC/MCFixedLenDisassembler.h" export * } + module MC_InstrItineraries { header "MC/MCInstrItineraries.h" export * } + module MC_Schedule { header "MC/MCSchedule.h" export * } + module MC_SubtargetFeature { header "MC/SubtargetFeature.h" export * } } module LLVM_Object { @@ -252,6 +259,13 @@ module LLVM_ProfileData { textual header "ProfileData/InstrProfData.inc" } +// FIXME: Mislayered? +module LLVM_Support_TargetRegistry { + requires cplusplus + header "Support/TargetRegistry.h" + export * +} + module LLVM_TableGen { requires cplusplus umbrella "TableGen" module * { export * } } module LLVM_Transforms { @@ -283,6 +297,7 @@ module LLVM_Utils { // These are intended for textual inclusion. textual header "Support/ARMTargetParser.def" textual header "Support/AArch64TargetParser.def" + textual header "Support/X86TargetParser.def" } // This part of the module is usable from both C and C++ code. @@ -305,3 +320,9 @@ module LLVM_Support_DataTypes_Src { header "llvm/Support/DataTypes.h" export * } + +module LLVM_WindowsManifest { + requires cplusplus + umbrella "WindowsManifest" + module * { export * } +} diff --git a/contrib/llvm/include/llvm/module.modulemap.build b/contrib/llvm/include/llvm/module.modulemap.build index 0f6f82af6e12..162a262a00a7 100644 --- a/contrib/llvm/include/llvm/module.modulemap.build +++ b/contrib/llvm/include/llvm/module.modulemap.build @@ -7,3 +7,7 @@ module LLVM_Config_ABI_Breaking { header "Config/abi-breaking.h" export * } +module LLVM_Config_Config { + header "Config/llvm-config.h" + export * +} |