diff options
Diffstat (limited to 'include')
56 files changed, 3344 insertions, 481 deletions
diff --git a/include/llvm/ADT/IntEqClasses.h b/include/llvm/ADT/IntEqClasses.h index 8e75c48e3764..0baee2f11a79 100644 --- a/include/llvm/ADT/IntEqClasses.h +++ b/include/llvm/ADT/IntEqClasses.h @@ -53,10 +53,10 @@ public: NumClasses = 0; } - /// join - Join the equivalence classes of a and b. After joining classes, - /// findLeader(a) == findLeader(b). - /// This requires an uncompressed map. - void join(unsigned a, unsigned b); + /// Join the equivalence classes of a and b. After joining classes, + /// findLeader(a) == findLeader(b). This requires an uncompressed map. + /// Returns the new leader. + unsigned join(unsigned a, unsigned b); /// findLeader - Compute the leader of a's equivalence class. This is the /// smallest member of the class. diff --git a/include/llvm/ADT/PointerEmbeddedInt.h b/include/llvm/ADT/PointerEmbeddedInt.h new file mode 100644 index 000000000000..8781d1803ac7 --- /dev/null +++ b/include/llvm/ADT/PointerEmbeddedInt.h @@ -0,0 +1,103 @@ +//===- llvm/ADT/PointerEmbeddedInt.h ----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_POINTEREMBEDDEDINT_H +#define LLVM_ADT_POINTEREMBEDDEDINT_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/Support/PointerLikeTypeTraits.h" +#include <climits> + +namespace llvm { + +/// Utility to embed an integer into a pointer-like type. This is specifically +/// intended to allow embedding integers where fewer bits are required than +/// exist in a pointer, and the integer can participate in abstractions along +/// side other pointer-like types. For example it can be placed into a \c +/// PointerSumType or \c PointerUnion. +/// +/// Note that much like pointers, an integer value of zero has special utility +/// due to boolean conversions. For example, a non-null value can be tested for +/// in the above abstractions without testing the particular active member. +/// Also, the default constructed value zero initializes the integer. +template <typename IntT, int Bits = sizeof(IntT) * CHAR_BIT> +class PointerEmbeddedInt { + uintptr_t Value; + + static_assert(Bits < sizeof(uintptr_t) * CHAR_BIT, + "Cannot embed more bits than we have in a pointer!"); + + enum : uintptr_t { + // We shift as many zeros into the value as we can while preserving the + // number of bits desired for the integer. + Shift = sizeof(uintptr_t) * CHAR_BIT - Bits, + + // We also want to be able to mask out the preserved bits for asserts. + Mask = static_cast<uintptr_t>(-1) << Bits + }; + + friend class PointerLikeTypeTraits<PointerEmbeddedInt>; + + explicit PointerEmbeddedInt(uintptr_t Value) : Value(Value) {} + +public: + PointerEmbeddedInt() : Value(0) {} + + PointerEmbeddedInt(IntT I) : Value(static_cast<uintptr_t>(I) << Shift) { + assert((I & Mask) == 0 && "Integer has bits outside those preserved!"); + } + + PointerEmbeddedInt &operator=(IntT I) { + assert((I & Mask) == 0 && "Integer has bits outside those preserved!"); + Value = static_cast<uintptr_t>(I) << Shift; + } + + // Note that this imilict conversion additionally allows all of the basic + // comparison operators to work transparently, etc. + operator IntT() const { return static_cast<IntT>(Value >> Shift); } +}; + +// Provide pointer like traits to support use with pointer unions and sum +// types. +template <typename IntT, int Bits> +class PointerLikeTypeTraits<PointerEmbeddedInt<IntT, Bits>> { + typedef PointerEmbeddedInt<IntT, Bits> T; + +public: + static inline void *getAsVoidPointer(const T &P) { + return reinterpret_cast<void *>(P.Value); + } + static inline T getFromVoidPointer(void *P) { + return T(reinterpret_cast<uintptr_t>(P)); + } + static inline T getFromVoidPointer(const void *P) { + return T(reinterpret_cast<uintptr_t>(P)); + } + + enum { NumLowBitsAvailable = T::Shift }; +}; + +// Teach DenseMap how to use PointerEmbeddedInt objects as keys if the Int type +// itself can be a key. +template <typename IntT, int Bits> +struct DenseMapInfo<PointerEmbeddedInt<IntT, Bits>> { + typedef PointerEmbeddedInt<IntT, Bits> T; + + typedef DenseMapInfo<IntT> IntInfo; + + static inline T getEmptyKey() { return IntInfo::getEmptyKey(); } + static inline T getTombstoneKey() { return IntInfo::getTombstoneKey(); } + static unsigned getHashValue(const T &Arg) { + return IntInfo::getHashValue(Arg); + } + static bool isEqual(const T &LHS, const T &RHS) { return LHS == RHS; } +}; +} + +#endif diff --git a/include/llvm/ADT/PointerIntPair.h b/include/llvm/ADT/PointerIntPair.h index 0058d85d1ae4..83fbf127e6da 100644 --- a/include/llvm/ADT/PointerIntPair.h +++ b/include/llvm/ADT/PointerIntPair.h @@ -55,20 +55,25 @@ public: PointerTy getPointer() const { return Info::getPointer(Value); } - IntType getInt() const { return (IntType)Info::getInt(Value); } + IntType getInt() const { + return (IntType)Info::getInt(Value); + } void setPointer(PointerTy PtrVal) { Value = Info::updatePointer(Value, PtrVal); } - void setInt(IntType IntVal) { Value = Info::updateInt(Value, IntVal); } + void setInt(IntType IntVal) { + Value = Info::updateInt(Value, static_cast<intptr_t>(IntVal)); + } void initWithPointer(PointerTy PtrVal) { Value = Info::updatePointer(0, PtrVal); } void setPointerAndInt(PointerTy PtrVal, IntType IntVal) { - Value = Info::updateInt(Info::updatePointer(0, PtrVal), IntVal); + Value = Info::updateInt(Info::updatePointer(0, PtrVal), + static_cast<intptr_t>(IntVal)); } PointerTy const *getAddrOfPointer() const { diff --git a/include/llvm/ADT/PointerSumType.h b/include/llvm/ADT/PointerSumType.h new file mode 100644 index 000000000000..6b8618fc5a17 --- /dev/null +++ b/include/llvm/ADT/PointerSumType.h @@ -0,0 +1,205 @@ +//===- llvm/ADT/PointerSumType.h --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_POINTERSUMTYPE_H +#define LLVM_ADT_POINTERSUMTYPE_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/PointerLikeTypeTraits.h" + +namespace llvm { + +/// A compile time pair of an integer tag and the pointer-like type which it +/// indexes within a sum type. Also allows the user to specify a particular +/// traits class for pointer types with custom behavior such as over-aligned +/// allocation. +template <uintptr_t N, typename PointerArgT, + typename TraitsArgT = PointerLikeTypeTraits<PointerArgT>> +struct PointerSumTypeMember { + enum { Tag = N }; + typedef PointerArgT PointerT; + typedef TraitsArgT TraitsT; +}; + +namespace detail { + +template <typename TagT, typename... MemberTs> +struct PointerSumTypeHelper; + +} + +/// A sum type over pointer-like types. +/// +/// This is a normal tagged union across pointer-like types that uses the low +/// bits of the pointers to store the tag. +/// +/// Each member of the sum type is specified by passing a \c +/// PointerSumTypeMember specialization in the variadic member argument list. +/// This allows the user to control the particular tag value associated with +/// a particular type, use the same type for multiple different tags, and +/// customize the pointer-like traits used for a particular member. Note that +/// these *must* be specializations of \c PointerSumTypeMember, no other type +/// will suffice, even if it provides a compatible interface. +/// +/// This type implements all of the comparison operators and even hash table +/// support by comparing the underlying storage of the pointer values. It +/// doesn't support delegating to particular members for comparisons. +/// +/// It also default constructs to a zero tag with a null pointer, whatever that +/// would be. This means that the zero value for the tag type is significant +/// and may be desireable to set to a state that is particularly desirable to +/// default construct. +/// +/// There is no support for constructing or accessing with a dynamic tag as +/// that would fundamentally violate the type safety provided by the sum type. +template <typename TagT, typename... MemberTs> class PointerSumType { + uintptr_t Value; + + typedef detail::PointerSumTypeHelper<TagT, MemberTs...> HelperT; + +public: + PointerSumType() : Value(0) {} + + /// A typed constructor for a specific tagged member of the sum type. + template <TagT N> + static PointerSumType + create(typename HelperT::template Lookup<N>::PointerT Pointer) { + PointerSumType Result; + void *V = HelperT::template Lookup<N>::TraitsT::getAsVoidPointer(Pointer); + assert((reinterpret_cast<uintptr_t>(V) & HelperT::TagMask) == 0 && + "Pointer is insufficiently aligned to store the discriminant!"); + Result.Value = reinterpret_cast<uintptr_t>(V) | N; + return Result; + } + + TagT getTag() const { return static_cast<TagT>(Value & HelperT::TagMask); } + + template <TagT N> bool is() const { return N == getTag(); } + + template <TagT N> typename HelperT::template Lookup<N>::PointerT get() const { + void *P = is<N>() ? getImpl() : nullptr; + return HelperT::template Lookup<N>::TraitsT::getFromVoidPointer(P); + } + + template <TagT N> + typename HelperT::template Lookup<N>::PointerT cast() const { + assert(is<N>() && "This instance has a different active member."); + return HelperT::template Lookup<N>::TraitsT::getFromVoidPointer(getImpl()); + } + + operator bool() const { return Value & HelperT::PointerMask; } + bool operator==(const PointerSumType &R) const { return Value == R.Value; } + bool operator!=(const PointerSumType &R) const { return Value != R.Value; } + bool operator<(const PointerSumType &R) const { return Value < R.Value; } + bool operator>(const PointerSumType &R) const { return Value > R.Value; } + bool operator<=(const PointerSumType &R) const { return Value <= R.Value; } + bool operator>=(const PointerSumType &R) const { return Value >= R.Value; } + + uintptr_t getOpaqueValue() const { return Value; } + +protected: + void *getImpl() const { + return reinterpret_cast<void *>(Value & HelperT::PointerMask); + } +}; + +namespace detail { + +/// A helper template for implementing \c PointerSumType. It provides fast +/// compile-time lookup of the member from a particular tag value, along with +/// useful constants and compile time checking infrastructure.. +template <typename TagT, typename... MemberTs> +struct PointerSumTypeHelper : MemberTs... { + // First we use a trick to allow quickly looking up information about + // a particular member of the sum type. This works because we arranged to + // have this type derive from all of the member type templates. We can select + // the matching member for a tag using type deduction during overload + // resolution. + template <TagT N, typename PointerT, typename TraitsT> + static PointerSumTypeMember<N, PointerT, TraitsT> + LookupOverload(PointerSumTypeMember<N, PointerT, TraitsT> *); + template <TagT N> static void LookupOverload(...); + template <TagT N> struct Lookup { + // Compute a particular member type by resolving the lookup helper ovorload. + typedef decltype(LookupOverload<N>( + static_cast<PointerSumTypeHelper *>(nullptr))) MemberT; + + /// The Nth member's pointer type. + typedef typename MemberT::PointerT PointerT; + + /// The Nth member's traits type. + typedef typename MemberT::TraitsT TraitsT; + }; + + // Next we need to compute the number of bits available for the discriminant + // by taking the min of the bits available for each member. Much of this + // would be amazingly easier with good constexpr support. + template <uintptr_t V, uintptr_t... Vs> + struct Min : std::integral_constant< + uintptr_t, (V < Min<Vs...>::value ? V : Min<Vs...>::value)> { + }; + template <uintptr_t V> + struct Min<V> : std::integral_constant<uintptr_t, V> {}; + enum { NumTagBits = Min<MemberTs::TraitsT::NumLowBitsAvailable...>::value }; + + // Also compute the smallest discriminant and various masks for convenience. + enum : uint64_t { + MinTag = Min<MemberTs::Tag...>::value, + PointerMask = static_cast<uint64_t>(-1) << NumTagBits, + TagMask = ~PointerMask + }; + + // Finally we need a recursive template to do static checks of each + // member. + template <typename MemberT, typename... InnerMemberTs> + struct Checker : Checker<InnerMemberTs...> { + static_assert(MemberT::Tag < (1 << NumTagBits), + "This discriminant value requires too many bits!"); + }; + template <typename MemberT> struct Checker<MemberT> : std::true_type { + static_assert(MemberT::Tag < (1 << NumTagBits), + "This discriminant value requires too many bits!"); + }; + static_assert(Checker<MemberTs...>::value, + "Each member must pass the checker."); +}; + +} + +// Teach DenseMap how to use PointerSumTypes as keys. +template <typename TagT, typename... MemberTs> +struct DenseMapInfo<PointerSumType<TagT, MemberTs...>> { + typedef PointerSumType<TagT, MemberTs...> SumType; + + typedef detail::PointerSumTypeHelper<TagT, MemberTs...> HelperT; + enum { SomeTag = HelperT::MinTag }; + typedef typename HelperT::template Lookup<HelperT::MinTag>::PointerT + SomePointerT; + typedef DenseMapInfo<SomePointerT> SomePointerInfo; + + static inline SumType getEmptyKey() { + return SumType::create<SomeTag>(SomePointerInfo::getEmptyKey()); + } + static inline SumType getTombstoneKey() { + return SumType::create<SomeTag>( + SomePointerInfo::getTombstoneKey()); + } + static unsigned getHashValue(const SumType &Arg) { + uintptr_t OpaqueValue = Arg.getOpaqueValue(); + return DenseMapInfo<uintptr_t>::getHashValue(OpaqueValue); + } + static bool isEqual(const SumType &LHS, const SumType &RHS) { + return LHS == RHS; + } +}; + +} + +#endif diff --git a/include/llvm/ADT/Twine.h b/include/llvm/ADT/Twine.h index db0bf4b68de8..81b1a6d946fc 100644 --- a/include/llvm/ADT/Twine.h +++ b/include/llvm/ADT/Twine.h @@ -101,15 +101,13 @@ namespace llvm { /// A pointer to a SmallString instance. SmallStringKind, - /// A char value reinterpreted as a pointer, to render as a character. + /// A char value, to render as a character. CharKind, - /// An unsigned int value reinterpreted as a pointer, to render as an - /// unsigned decimal integer. + /// An unsigned int value, to render as an unsigned decimal integer. DecUIKind, - /// An int value reinterpreted as a pointer, to render as a signed - /// decimal integer. + /// An int value, to render as a signed decimal integer. DecIKind, /// A pointer to an unsigned long value, to render as an unsigned decimal diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h index ef3d5e8fe3df..e02f3ab2de1f 100644 --- a/include/llvm/Analysis/LazyCallGraph.h +++ b/include/llvm/Analysis/LazyCallGraph.h @@ -104,54 +104,10 @@ class LazyCallGraph { public: class Node; class SCC; + class iterator; typedef SmallVector<PointerUnion<Function *, Node *>, 4> NodeVectorT; typedef SmallVectorImpl<PointerUnion<Function *, Node *>> NodeVectorImplT; - /// A lazy iterator used for both the entry nodes and child nodes. - /// - /// When this iterator is dereferenced, if not yet available, a function will - /// be scanned for "calls" or uses of functions and its child information - /// will be constructed. All of these results are accumulated and cached in - /// the graph. - class iterator - : public iterator_adaptor_base<iterator, NodeVectorImplT::iterator, - std::forward_iterator_tag, Node> { - friend class LazyCallGraph; - friend class LazyCallGraph::Node; - - LazyCallGraph *G; - NodeVectorImplT::iterator E; - - // Build the iterator for a specific position in a node list. - iterator(LazyCallGraph &G, NodeVectorImplT::iterator NI, - NodeVectorImplT::iterator E) - : iterator_adaptor_base(NI), G(&G), E(E) { - while (I != E && I->isNull()) - ++I; - } - - public: - iterator() {} - - using iterator_adaptor_base::operator++; - iterator &operator++() { - do { - ++I; - } while (I != E && I->isNull()); - return *this; - } - - reference operator*() const { - if (I->is<Node *>()) - return *I->get<Node *>(); - - Function *F = I->get<Function *>(); - Node &ChildN = G->get(*F); - *I = &ChildN; - return ChildN; - } - }; - /// A node in the call graph. /// /// This represents a single node. It's primary roles are to cache the list of @@ -200,6 +156,51 @@ public: bool operator!=(const Node &N) const { return !operator==(N); } }; + /// A lazy iterator used for both the entry nodes and child nodes. + /// + /// When this iterator is dereferenced, if not yet available, a function will + /// be scanned for "calls" or uses of functions and its child information + /// will be constructed. All of these results are accumulated and cached in + /// the graph. + class iterator + : public iterator_adaptor_base<iterator, NodeVectorImplT::iterator, + std::forward_iterator_tag, Node> { + friend class LazyCallGraph; + friend class LazyCallGraph::Node; + + LazyCallGraph *G; + NodeVectorImplT::iterator E; + + // Build the iterator for a specific position in a node list. + iterator(LazyCallGraph &G, NodeVectorImplT::iterator NI, + NodeVectorImplT::iterator E) + : iterator_adaptor_base(NI), G(&G), E(E) { + while (I != E && I->isNull()) + ++I; + } + + public: + iterator() {} + + using iterator_adaptor_base::operator++; + iterator &operator++() { + do { + ++I; + } while (I != E && I->isNull()); + return *this; + } + + reference operator*() const { + if (I->is<Node *>()) + return *I->get<Node *>(); + + Function *F = I->get<Function *>(); + Node &ChildN = G->get(*F); + *I = &ChildN; + return ChildN; + } + }; + /// An SCC of the call graph. /// /// This represents a Strongly Connected Component of the call graph as diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h index c219bd85a48a..70e636ce1f3d 100644 --- a/include/llvm/Analysis/LoopInfo.h +++ b/include/llvm/Analysis/LoopInfo.h @@ -59,38 +59,37 @@ template<class N, class M> class LoopInfoBase; template<class N, class M> class LoopBase; //===----------------------------------------------------------------------===// -/// LoopBase class - Instances of this class are used to represent loops that -/// are detected in the flow graph +/// Instances of this class are used to represent loops that are detected in the +/// flow graph. /// template<class BlockT, class LoopT> class LoopBase { LoopT *ParentLoop; - // SubLoops - Loops contained entirely within this one. + // Loops contained entirely within this one. std::vector<LoopT *> SubLoops; - // Blocks - The list of blocks in this loop. First entry is the header node. + // The list of blocks in this loop. First entry is the header node. std::vector<BlockT*> Blocks; SmallPtrSet<const BlockT*, 8> DenseBlockSet; - /// Indicator that this loops has been "unlooped", so there's no loop here - /// anymore. - bool IsUnloop = false; + /// Indicator that this loop is no longer a valid loop. + bool IsInvalid = false; LoopBase(const LoopBase<BlockT, LoopT> &) = delete; const LoopBase<BlockT, LoopT>& operator=(const LoopBase<BlockT, LoopT> &) = delete; public: - /// Loop ctor - This creates an empty loop. + /// This creates an empty loop. LoopBase() : ParentLoop(nullptr) {} ~LoopBase() { for (size_t i = 0, e = SubLoops.size(); i != e; ++i) delete SubLoops[i]; } - /// getLoopDepth - Return the nesting level of this loop. An outer-most - /// loop has depth 1, for consistency with loop depth values used for basic - /// blocks, where depth 0 is used for blocks not inside any loops. + /// Return the nesting level of this loop. An outer-most loop has depth 1, + /// for consistency with loop depth values used for basic blocks, where depth + /// 0 is used for blocks not inside any loops. unsigned getLoopDepth() const { unsigned D = 1; for (const LoopT *CurLoop = ParentLoop; CurLoop; @@ -101,33 +100,28 @@ public: BlockT *getHeader() const { return Blocks.front(); } LoopT *getParentLoop() const { return ParentLoop; } - /// setParentLoop is a raw interface for bypassing addChildLoop. + /// This is a raw interface for bypassing addChildLoop. void setParentLoop(LoopT *L) { ParentLoop = L; } - /// contains - Return true if the specified loop is contained within in - /// this loop. - /// + /// Return true if the specified loop is contained within in this loop. bool contains(const LoopT *L) const { if (L == this) return true; if (!L) return false; return contains(L->getParentLoop()); } - /// contains - Return true if the specified basic block is in this loop. - /// + /// Return true if the specified basic block is in this loop. bool contains(const BlockT *BB) const { return DenseBlockSet.count(BB); } - /// contains - Return true if the specified instruction is in this loop. - /// + /// Return true if the specified instruction is in this loop. template<class InstT> bool contains(const InstT *Inst) const { return contains(Inst->getParent()); } - /// iterator/begin/end - Return the loops contained entirely within this loop. - /// + /// Return the loops contained entirely within this loop. const std::vector<LoopT *> &getSubLoops() const { return SubLoops; } std::vector<LoopT *> &getSubLoopsVector() { return SubLoops; } typedef typename std::vector<LoopT *>::const_iterator iterator; @@ -139,8 +133,7 @@ public: reverse_iterator rend() const { return SubLoops.rend(); } bool empty() const { return SubLoops.empty(); } - /// getBlocks - Get a list of the basic blocks which make up this loop. - /// + /// Get a list of the basic blocks which make up this loop. const std::vector<BlockT*> &getBlocks() const { return Blocks; } typedef typename std::vector<BlockT*>::const_iterator block_iterator; block_iterator block_begin() const { return Blocks.begin(); } @@ -149,21 +142,19 @@ public: return make_range(block_begin(), block_end()); } - /// getNumBlocks - Get the number of blocks in this loop in constant time. + /// Get the number of blocks in this loop in constant time. unsigned getNumBlocks() const { return Blocks.size(); } - /// Mark this loop as having been unlooped - the last backedge was removed and - /// we no longer have a loop. - void markUnlooped() { IsUnloop = true; } + /// Invalidate the loop, indicating that it is no longer a loop. + void invalidate() { IsInvalid = true; } - /// Return true if this no longer represents a loop. - bool isUnloop() const { return IsUnloop; } + /// Return true if this loop is no longer valid. + bool isInvalid() { return IsInvalid; } - /// isLoopExiting - True if terminator in the block can branch to another - /// block that is outside of the current loop. - /// + /// True if terminator in the block can branch to another block that is + /// outside of the current loop. bool isLoopExiting(const BlockT *BB) const { typedef GraphTraits<const BlockT*> BlockTraits; for (typename BlockTraits::ChildIteratorType SI = @@ -175,8 +166,7 @@ public: return false; } - /// getNumBackEdges - Calculate the number of back edges to the loop header - /// + /// Calculate the number of back edges to the loop header. unsigned getNumBackEdges() const { unsigned NumBackEdges = 0; BlockT *H = getHeader(); @@ -199,53 +189,49 @@ public: // induction variable canonicalization pass should be used to normalize loops // for easy analysis. These methods assume canonical loops. - /// getExitingBlocks - Return all blocks inside the loop that have successors - /// outside of the loop. These are the blocks _inside of the current loop_ - /// which branch out. The returned list is always unique. - /// + /// Return all blocks inside the loop that have successors outside of the + /// loop. These are the blocks _inside of the current loop_ which branch out. + /// The returned list is always unique. void getExitingBlocks(SmallVectorImpl<BlockT *> &ExitingBlocks) const; - /// getExitingBlock - If getExitingBlocks would return exactly one block, - /// return that block. Otherwise return null. + /// If getExitingBlocks would return exactly one block, return that block. + /// Otherwise return null. BlockT *getExitingBlock() const; - /// getExitBlocks - Return all of the successor blocks of this loop. These - /// are the blocks _outside of the current loop_ which are branched to. - /// + /// Return all of the successor blocks of this loop. These are the blocks + /// _outside of the current loop_ which are branched to. void getExitBlocks(SmallVectorImpl<BlockT*> &ExitBlocks) const; - /// getExitBlock - If getExitBlocks would return exactly one block, - /// return that block. Otherwise return null. + /// If getExitBlocks would return exactly one block, return that block. + /// Otherwise return null. BlockT *getExitBlock() const; /// Edge type. typedef std::pair<const BlockT*, const BlockT*> Edge; - /// getExitEdges - Return all pairs of (_inside_block_,_outside_block_). + /// Return all pairs of (_inside_block_,_outside_block_). void getExitEdges(SmallVectorImpl<Edge> &ExitEdges) const; - /// getLoopPreheader - If there is a preheader for this loop, return it. A - /// loop has a preheader if there is only one edge to the header of the loop - /// from outside of the loop. If this is the case, the block branching to the - /// header of the loop is the preheader node. + /// If there is a preheader for this loop, return it. A loop has a preheader + /// if there is only one edge to the header of the loop from outside of the + /// loop. If this is the case, the block branching to the header of the loop + /// is the preheader node. /// /// This method returns null if there is no preheader for the loop. - /// BlockT *getLoopPreheader() const; - /// getLoopPredecessor - If the given loop's header has exactly one unique - /// predecessor outside the loop, return it. Otherwise return null. - /// This is less strict that the loop "preheader" concept, which requires + /// If the given loop's header has exactly one unique predecessor outside the + /// loop, return it. Otherwise return null. + /// This is less strict that the loop "preheader" concept, which requires /// the predecessor to have exactly one successor. - /// BlockT *getLoopPredecessor() const; - /// getLoopLatch - If there is a single latch block for this loop, return it. + /// If there is a single latch block for this loop, return it. /// A latch block is a block that contains a branch back to the header. BlockT *getLoopLatch() const; - /// getLoopLatches - Return all loop latch blocks of this loop. A latch block - /// is a block that contains a branch back to the header. + /// Return all loop latch blocks of this loop. A latch block is a block that + /// contains a branch back to the header. void getLoopLatches(SmallVectorImpl<BlockT *> &LoopLatches) const { BlockT *H = getHeader(); typedef GraphTraits<Inverse<BlockT*> > InvBlockTraits; @@ -260,32 +246,29 @@ public: // APIs for updating loop information after changing the CFG // - /// addBasicBlockToLoop - This method is used by other analyses to update loop - /// information. NewBB is set to be a new member of the current loop. + /// This method is used by other analyses to update loop information. + /// NewBB is set to be a new member of the current loop. /// Because of this, it is added as a member of all parent loops, and is added /// to the specified LoopInfo object as being in the current basic block. It /// is not valid to replace the loop header with this method. - /// void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase<BlockT, LoopT> &LI); - /// replaceChildLoopWith - This is used when splitting loops up. It replaces - /// the OldChild entry in our children list with NewChild, and updates the - /// parent pointer of OldChild to be null and the NewChild to be this loop. + /// This is used when splitting loops up. It replaces the OldChild entry in + /// our children list with NewChild, and updates the parent pointer of + /// OldChild to be null and the NewChild to be this loop. /// This updates the loop depth of the new child. void replaceChildLoopWith(LoopT *OldChild, LoopT *NewChild); - /// addChildLoop - Add the specified loop to be a child of this loop. This - /// updates the loop depth of the new child. - /// + /// Add the specified loop to be a child of this loop. + /// This updates the loop depth of the new child. void addChildLoop(LoopT *NewChild) { assert(!NewChild->ParentLoop && "NewChild already has a parent!"); NewChild->ParentLoop = static_cast<LoopT *>(this); SubLoops.push_back(NewChild); } - /// removeChildLoop - This removes the specified child from being a subloop of - /// this loop. The loop is not deleted, as it will presumably be inserted - /// into another loop. + /// This removes the specified child from being a subloop of this loop. The + /// loop is not deleted, as it will presumably be inserted into another loop. LoopT *removeChildLoop(iterator I) { assert(I != SubLoops.end() && "Cannot remove end iterator!"); LoopT *Child = *I; @@ -295,7 +278,7 @@ public: return Child; } - /// addBlockEntry - This adds a basic block directly to the basic block list. + /// This adds a basic block directly to the basic block list. /// This should only be used by transformations that create new loops. Other /// transformations should use addBasicBlockToLoop. void addBlockEntry(BlockT *BB) { @@ -303,19 +286,18 @@ public: DenseBlockSet.insert(BB); } - /// reverseBlocks - interface to reverse Blocks[from, end of loop] in this loop + /// interface to reverse Blocks[from, end of loop] in this loop void reverseBlock(unsigned from) { std::reverse(Blocks.begin() + from, Blocks.end()); } - /// reserveBlocks- interface to do reserve() for Blocks + /// interface to do reserve() for Blocks void reserveBlocks(unsigned size) { Blocks.reserve(size); } - /// moveToHeader - This method is used to move BB (which must be part of this - /// loop) to be the loop header of the loop (the block that dominates all - /// others). + /// This method is used to move BB (which must be part of this loop) to be the + /// loop header of the loop (the block that dominates all others). void moveToHeader(BlockT *BB) { if (Blocks[0] == BB) return; for (unsigned i = 0; ; ++i) { @@ -328,9 +310,9 @@ public: } } - /// removeBlockFromLoop - This removes the specified basic block from the - /// current loop, updating the Blocks as appropriate. This does not update - /// the mapping in the LoopInfo class. + /// This removes the specified basic block from the current loop, updating the + /// Blocks as appropriate. This does not update the mapping in the LoopInfo + /// class. void removeBlockFromLoop(BlockT *BB) { auto I = std::find(Blocks.begin(), Blocks.end(), BB); assert(I != Blocks.end() && "N is not in this list!"); @@ -339,10 +321,10 @@ public: DenseBlockSet.erase(BB); } - /// verifyLoop - Verify loop structure + /// Verify loop structure void verifyLoop() const; - /// verifyLoop - Verify loop structure of this loop and all nested loops. + /// Verify loop structure of this loop and all nested loops. void verifyLoopNest(DenseSet<const LoopT*> *Loops) const; void print(raw_ostream &OS, unsigned Depth = 0) const; @@ -368,28 +350,26 @@ class Loop : public LoopBase<BasicBlock, Loop> { public: Loop() {} - /// isLoopInvariant - Return true if the specified value is loop invariant - /// + /// Return true if the specified value is loop invariant. bool isLoopInvariant(const Value *V) const; - /// hasLoopInvariantOperands - Return true if all the operands of the - /// specified instruction are loop invariant. + /// Return true if all the operands of the specified instruction are loop + /// invariant. bool hasLoopInvariantOperands(const Instruction *I) const; - /// makeLoopInvariant - If the given value is an instruction inside of the - /// loop and it can be hoisted, do so to make it trivially loop-invariant. + /// If the given value is an instruction inside of the loop and it can be + /// hoisted, do so to make it trivially loop-invariant. /// Return true if the value after any hoisting is loop invariant. This /// function can be used as a slightly more aggressive replacement for /// isLoopInvariant. /// /// If InsertPt is specified, it is the point to hoist instructions to. /// If null, the terminator of the loop preheader is used. - /// bool makeLoopInvariant(Value *V, bool &Changed, Instruction *InsertPt = nullptr) const; - /// makeLoopInvariant - If the given instruction is inside of the - /// loop and it can be hoisted, do so to make it trivially loop-invariant. + /// If the given instruction is inside of the loop and it can be hoisted, do + /// so to make it trivially loop-invariant. /// Return true if the instruction after any hoisting is loop invariant. This /// function can be used as a slightly more aggressive replacement for /// isLoopInvariant. @@ -400,28 +380,26 @@ public: bool makeLoopInvariant(Instruction *I, bool &Changed, Instruction *InsertPt = nullptr) const; - /// getCanonicalInductionVariable - Check to see if the loop has a canonical - /// induction variable: an integer recurrence that starts at 0 and increments - /// by one each time through the loop. If so, return the phi node that - /// corresponds to it. + /// Check to see if the loop has a canonical induction variable: an integer + /// recurrence that starts at 0 and increments by one each time through the + /// loop. If so, return the phi node that corresponds to it. /// /// The IndVarSimplify pass transforms loops to have a canonical induction /// variable. /// PHINode *getCanonicalInductionVariable() const; - /// isLCSSAForm - Return true if the Loop is in LCSSA form + /// Return true if the Loop is in LCSSA form. bool isLCSSAForm(DominatorTree &DT) const; - /// \brief Return true if this Loop and all inner subloops are in LCSSA form. + /// Return true if this Loop and all inner subloops are in LCSSA form. bool isRecursivelyLCSSAForm(DominatorTree &DT) const; - /// isLoopSimplifyForm - Return true if the Loop is in the form that - /// the LoopSimplify form transforms loops to, which is sometimes called - /// normal form. + /// Return true if the Loop is in the form that the LoopSimplify form + /// transforms loops to, which is sometimes called normal form. bool isLoopSimplifyForm() const; - /// isSafeToClone - Return true if the loop body is safe to clone in practice. + /// Return true if the loop body is safe to clone in practice. bool isSafeToClone() const; /// Returns true if the loop is annotated parallel. @@ -454,23 +432,22 @@ public: /// operand should should be the node itself. void setLoopID(MDNode *LoopID) const; - /// hasDedicatedExits - Return true if no exit block for the loop - /// has a predecessor that is outside the loop. + /// Return true if no exit block for the loop has a predecessor that is + /// outside the loop. bool hasDedicatedExits() const; - /// getUniqueExitBlocks - Return all unique successor blocks of this loop. + /// Return all unique successor blocks of this loop. /// These are the blocks _outside of the current loop_ which are branched to. /// This assumes that loop exits are in canonical form. - /// void getUniqueExitBlocks(SmallVectorImpl<BasicBlock *> &ExitBlocks) const; - /// getUniqueExitBlock - If getUniqueExitBlocks would return exactly one - /// block, return that block. Otherwise return null. + /// If getUniqueExitBlocks would return exactly one block, return that block. + /// Otherwise return null. BasicBlock *getUniqueExitBlock() const; void dump() const; - /// \brief Return the debug location of the start of this loop. + /// Return the debug location of the start of this loop. /// This looks for a BB terminating instruction with a known debug /// location by looking at the preheader and header blocks. If it /// cannot find a terminating instruction with location information, @@ -498,7 +475,7 @@ private: }; //===----------------------------------------------------------------------===// -/// LoopInfo - This class builds and contains all of the top level loop +/// This class builds and contains all of the top-level loop /// structures in the specified function. /// @@ -507,6 +484,8 @@ class LoopInfoBase { // BBMap - Mapping of basic blocks to the inner most loop they occur in DenseMap<const BlockT *, LoopT *> BBMap; std::vector<LoopT *> TopLevelLoops; + std::vector<LoopT *> RemovedLoops; + friend class LoopBase<BlockT, LoopT>; friend class LoopInfo; @@ -538,6 +517,9 @@ public: for (auto *L : TopLevelLoops) delete L; TopLevelLoops.clear(); + for (auto *L : RemovedLoops) + delete L; + RemovedLoops.clear(); } /// iterator/begin/end - The interface to the top-level loops in the current @@ -552,33 +534,30 @@ public: reverse_iterator rend() const { return TopLevelLoops.rend(); } bool empty() const { return TopLevelLoops.empty(); } - /// getLoopFor - Return the inner most loop that BB lives in. If a basic - /// block is in no loop (for example the entry node), null is returned. - /// + /// Return the inner most loop that BB lives in. If a basic block is in no + /// loop (for example the entry node), null is returned. LoopT *getLoopFor(const BlockT *BB) const { return BBMap.lookup(BB); } - /// operator[] - same as getLoopFor... - /// + /// Same as getLoopFor. const LoopT *operator[](const BlockT *BB) const { return getLoopFor(BB); } - /// getLoopDepth - Return the loop nesting level of the specified block. A - /// depth of 0 means the block is not inside any loop. - /// + /// Return the loop nesting level of the specified block. A depth of 0 means + /// the block is not inside any loop. unsigned getLoopDepth(const BlockT *BB) const { const LoopT *L = getLoopFor(BB); return L ? L->getLoopDepth() : 0; } - // isLoopHeader - True if the block is a loop header node + // True if the block is a loop header node bool isLoopHeader(const BlockT *BB) const { const LoopT *L = getLoopFor(BB); return L && L->getHeader() == BB; } - /// removeLoop - This removes the specified top-level loop from this loop info - /// object. The loop is not deleted, as it will presumably be inserted into + /// This removes the specified top-level loop from this loop info object. + /// The loop is not deleted, as it will presumably be inserted into /// another loop. LoopT *removeLoop(iterator I) { assert(I != end() && "Cannot remove end iterator!"); @@ -588,9 +567,9 @@ public: return L; } - /// changeLoopFor - Change the top-level loop that contains BB to the - /// specified loop. This should be used by transformations that restructure - /// the loop hierarchy tree. + /// Change the top-level loop that contains BB to the specified loop. + /// This should be used by transformations that restructure the loop hierarchy + /// tree. void changeLoopFor(BlockT *BB, LoopT *L) { if (!L) { BBMap.erase(BB); @@ -599,8 +578,8 @@ public: BBMap[BB] = L; } - /// changeTopLevelLoop - Replace the specified loop in the top-level loops - /// list with the indicated loop. + /// Replace the specified loop in the top-level loops list with the indicated + /// loop. void changeTopLevelLoop(LoopT *OldLoop, LoopT *NewLoop) { auto I = std::find(TopLevelLoops.begin(), TopLevelLoops.end(), OldLoop); @@ -610,14 +589,13 @@ public: "Loops already embedded into a subloop!"); } - /// addTopLevelLoop - This adds the specified loop to the collection of - /// top-level loops. + /// This adds the specified loop to the collection of top-level loops. void addTopLevelLoop(LoopT *New) { assert(!New->getParentLoop() && "Loop already in subloop!"); TopLevelLoops.push_back(New); } - /// removeBlock - This method completely removes BB from all data structures, + /// This method completely removes BB from all data structures, /// including all of the Loop objects it is nested in and our mapping from /// BasicBlocks to loops. void removeBlock(BlockT *BB) { @@ -670,15 +648,14 @@ public: // Most of the public interface is provided via LoopInfoBase. - /// updateUnloop - Update LoopInfo after removing the last backedge from a - /// loop--now the "unloop". This updates the loop forest and parent loops for - /// each block so that Unloop is no longer referenced, but does not actually - /// delete the Unloop object. Generally, the loop pass manager should manage - /// deleting the Unloop. - void updateUnloop(Loop *Unloop); + /// Update LoopInfo after removing the last backedge from a loop. This updates + /// the loop forest and parent loops for each block so that \c L is no longer + /// referenced, but does not actually delete \c L immediately. The pointer + /// will remain valid until this LoopInfo's memory is released. + void markAsRemoved(Loop *L); - /// replacementPreservesLCSSAForm - Returns true if replacing From with To - /// everywhere is guaranteed to preserve LCSSA form. + /// Returns true if replacing From with To everywhere is guaranteed to + /// preserve LCSSA form. bool replacementPreservesLCSSAForm(Instruction *From, Value *To) { // Preserving LCSSA form is only problematic if the replacing value is an // instruction. @@ -698,8 +675,7 @@ public: return ToLoop->contains(getLoopFor(From->getParent())); } - /// \brief Checks if moving a specific instruction can break LCSSA in any - /// loop. + /// Checks if moving a specific instruction can break LCSSA in any loop. /// /// Return true if moving \p Inst to before \p NewLoc will break LCSSA, /// assuming that the function containing \p Inst and \p NewLoc is currently diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index f5e778b2f262..cf29fc9ef889 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -259,7 +259,7 @@ public: void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const; /// Lower the specified LLVM Constant to an MCExpr. - const MCExpr *lowerConstant(const Constant *CV); + virtual const MCExpr *lowerConstant(const Constant *CV); /// \brief Print a general LLVM constant to the .s file. void EmitGlobalConstant(const DataLayout &DL, const Constant *CV); diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h index fa612d981dec..72b3adc7de99 100644 --- a/include/llvm/CodeGen/DIE.h +++ b/include/llvm/CodeGen/DIE.h @@ -29,6 +29,48 @@ class MCSymbol; class raw_ostream; class DwarfTypeUnit; +// AsmStreamerBase - A base abstract interface class defines methods that +// can be implemented to stream objects or can be implemented to +// calculate the size of the streamed objects. +// The derived classes will use an AsmPrinter to implement the methods. +// +// TODO: complete this interface and use it to merge EmitValue and SizeOf +// methods in the DIE classes below. +class AsmStreamerBase { +protected: + const AsmPrinter *AP; + AsmStreamerBase(const AsmPrinter *AP) : AP(AP) {} + +public: + virtual ~AsmStreamerBase() {} + virtual unsigned emitULEB128(uint64_t Value, const char *Desc = nullptr, + unsigned PadTo = 0) = 0; + virtual unsigned emitInt8(unsigned char Value) = 0; + virtual unsigned emitBytes(StringRef Data) = 0; +}; + +/// EmittingAsmStreamer - Implements AbstractAsmStreamer to stream objects. +/// Notice that the return value is not the actual size of the streamed object. +/// For size calculation use SizeReporterAsmStreamer. +class EmittingAsmStreamer : public AsmStreamerBase { +public: + EmittingAsmStreamer(const AsmPrinter *AP) : AsmStreamerBase(AP) {} + unsigned emitULEB128(uint64_t Value, const char *Desc = nullptr, + unsigned PadTo = 0) override; + unsigned emitInt8(unsigned char Value) override; + unsigned emitBytes(StringRef Data) override; +}; + +/// SizeReporterAsmStreamer - Only reports the size of the streamed objects. +class SizeReporterAsmStreamer : public AsmStreamerBase { +public: + SizeReporterAsmStreamer(const AsmPrinter *AP) : AsmStreamerBase(AP) {} + unsigned emitULEB128(uint64_t Value, const char *Desc = nullptr, + unsigned PadTo = 0) override; + unsigned emitInt8(unsigned char Value) override; + unsigned emitBytes(StringRef Data) override; +}; + //===--------------------------------------------------------------------===// /// DIEAbbrevData - Dwarf abbreviation data, describes one attribute of a /// Dwarf abbreviation. diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index 0157bf9117e5..edade3164a3c 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -848,9 +848,9 @@ namespace llvm { public: explicit ConnectedVNInfoEqClasses(LiveIntervals &lis) : LIS(lis) {} - /// Classify - Classify the values in LI into connected components. - /// Return the number of connected components. - unsigned Classify(const LiveInterval *LI); + /// Classify the values in \p LR into connected components. + /// Returns the number of connected components. + unsigned Classify(const LiveRange &LR); /// getEqClass - Classify creates equivalence classes numbered 0..N. Return /// the equivalence class assigned the VNI. diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index 987634fb36c3..9bbdf3e071bd 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -141,6 +141,28 @@ public: LLVM_DUMP_METHOD void dump(const TargetRegisterInfo &TRI) const; }; +/// List of registers defined and used by a machine instruction. +class RegisterOperands { +public: + /// List of virtual regiserts and register units read by the instruction. + SmallVector<unsigned, 8> Uses; + /// \brief List of virtual registers and register units defined by the + /// instruction which are not dead. + SmallVector<unsigned, 8> Defs; + /// \brief List of virtual registers and register units defined by the + /// instruction but dead. + SmallVector<unsigned, 8> DeadDefs; + + /// Analyze the given instruction \p MI and fill in the Uses, Defs and + /// DeadDefs list based on the MachineOperand flags. + void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI, bool IgnoreDead = false); + + /// Use liveness information to find dead defs not marked with a dead flag + /// and move them to the DeadDefs vector. + void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS); +}; + /// Array of PressureDiffs. class PressureDiffs { PressureDiff *PDiffArray; @@ -161,6 +183,10 @@ public: const PressureDiff &operator[](unsigned Idx) const { return const_cast<PressureDiffs*>(this)->operator[](Idx); } + /// \brief Record pressure difference induced by the given operand list to + /// node with index \p Idx. + void addInstruction(unsigned Idx, const RegisterOperands &RegOpers, + const MachineRegisterInfo &MRI); }; /// Store the effects of a change in pressure on things that MI scheduler cares @@ -329,8 +355,17 @@ public: void setPos(MachineBasicBlock::const_iterator Pos) { CurrPos = Pos; } /// Recede across the previous instruction. - void recede(SmallVectorImpl<unsigned> *LiveUses = nullptr, - PressureDiff *PDiff = nullptr); + void recede(SmallVectorImpl<unsigned> *LiveUses = nullptr); + + /// Recede across the previous instruction. + /// This "low-level" variant assumes that recedeSkipDebugValues() was + /// called previously and takes precomputed RegisterOperands for the + /// instruction. + void recede(const RegisterOperands &RegOpers, + SmallVectorImpl<unsigned> *LiveUses = nullptr); + + /// Recede until we find an instruction which is not a DebugValue. + void recedeSkipDebugValues(); /// Advance across the current instruction. void advance(); diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h index f6ad7a8572ab..46c1029f62cf 100644 --- a/include/llvm/CodeGen/WinEHFuncInfo.h +++ b/include/llvm/CodeGen/WinEHFuncInfo.h @@ -93,8 +93,6 @@ struct WinEHFuncInfo { DenseMap<const Instruction *, int> EHPadStateMap; DenseMap<const FuncletPadInst *, int> FuncletBaseStateMap; DenseMap<const InvokeInst *, int> InvokeStateMap; - DenseMap<const CatchReturnInst *, const BasicBlock *> - CatchRetSuccessorColorMap; DenseMap<MCSymbol *, std::pair<int, MCSymbol *>> LabelToStateMap; SmallVector<CxxUnwindMapEntry, 4> CxxUnwindMap; SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap; @@ -125,8 +123,5 @@ void calculateSEHStateNumbers(const Function *ParentFn, WinEHFuncInfo &FuncInfo); void calculateClrEHStateNumbers(const Function *Fn, WinEHFuncInfo &FuncInfo); - -void calculateCatchReturnSuccessorColors(const Function *Fn, - WinEHFuncInfo &FuncInfo); } #endif // LLVM_CODEGEN_WINEHFUNCINFO_H diff --git a/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/include/llvm/DebugInfo/Symbolize/DIPrinter.h index 0703fb14da61..3098199bb4da 100644 --- a/include/llvm/DebugInfo/Symbolize/DIPrinter.h +++ b/include/llvm/DebugInfo/Symbolize/DIPrinter.h @@ -28,13 +28,16 @@ class DIPrinter { raw_ostream &OS; bool PrintFunctionNames; bool PrintPretty; - void printName(const DILineInfo &Info, bool Inlined); + int PrintSourceContext; + + void print(const DILineInfo &Info, bool Inlined); + void printContext(std::string FileName, int64_t Line); public: DIPrinter(raw_ostream &OS, bool PrintFunctionNames = true, - bool PrintPretty = false) + bool PrintPretty = false, int PrintSourceContext = 0) : OS(OS), PrintFunctionNames(PrintFunctionNames), - PrintPretty(PrintPretty) {} + PrintPretty(PrintPretty), PrintSourceContext(PrintSourceContext) {} DIPrinter &operator<<(const DILineInfo &Info); DIPrinter &operator<<(const DIInliningInfo &Info); diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 7dab5d1bc67f..84af4728b350 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -19,7 +19,6 @@ #include "LambdaResolver.h" #include "LogicalDylib.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Transforms/Utils/Cloning.h" #include <list> #include <memory> @@ -61,31 +60,36 @@ private: typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; - class ModuleOwner { + // Provide type-erasure for the Modules and MemoryManagers. + template <typename ResourceT> + class ResourceOwner { public: - ModuleOwner() = default; - ModuleOwner(const ModuleOwner&) = delete; - ModuleOwner& operator=(const ModuleOwner&) = delete; - virtual ~ModuleOwner() { } - virtual Module& getModule() const = 0; + ResourceOwner() = default; + ResourceOwner(const ResourceOwner&) = delete; + ResourceOwner& operator=(const ResourceOwner&) = delete; + virtual ~ResourceOwner() { } + virtual ResourceT& getResource() const = 0; }; - template <typename ModulePtrT> - class ModuleOwnerImpl : public ModuleOwner { + template <typename ResourceT, typename ResourcePtrT> + class ResourceOwnerImpl : public ResourceOwner<ResourceT> { public: - ModuleOwnerImpl(ModulePtrT ModulePtr) : ModulePtr(std::move(ModulePtr)) {} - Module& getModule() const override { return *ModulePtr; } + ResourceOwnerImpl(ResourcePtrT ResourcePtr) + : ResourcePtr(std::move(ResourcePtr)) {} + ResourceT& getResource() const override { return *ResourcePtr; } private: - ModulePtrT ModulePtr; + ResourcePtrT ResourcePtr; }; - template <typename ModulePtrT> - std::unique_ptr<ModuleOwner> wrapOwnership(ModulePtrT ModulePtr) { - return llvm::make_unique<ModuleOwnerImpl<ModulePtrT>>(std::move(ModulePtr)); + template <typename ResourceT, typename ResourcePtrT> + std::unique_ptr<ResourceOwner<ResourceT>> + wrapOwnership(ResourcePtrT ResourcePtr) { + typedef ResourceOwnerImpl<ResourceT, ResourcePtrT> RO; + return llvm::make_unique<RO>(std::move(ResourcePtr)); } struct LogicalModuleResources { - std::unique_ptr<ModuleOwner> SourceModuleOwner; + std::unique_ptr<ResourceOwner<Module>> SourceModule; std::set<const Function*> StubsToClone; std::unique_ptr<IndirectStubsMgrT> StubsMgr; @@ -93,15 +97,16 @@ private: // Explicit move constructor to make MSVC happy. LogicalModuleResources(LogicalModuleResources &&Other) - : SourceModuleOwner(std::move(Other.SourceModuleOwner)), + : SourceModule(std::move(Other.SourceModule)), StubsToClone(std::move(Other.StubsToClone)), StubsMgr(std::move(Other.StubsMgr)) {} // Explicit move assignment to make MSVC happy. LogicalModuleResources& operator=(LogicalModuleResources &&Other) { - SourceModuleOwner = std::move(Other.SourceModuleOwner); + SourceModule = std::move(Other.SourceModule); StubsToClone = std::move(Other.StubsToClone); StubsMgr = std::move(Other.StubsMgr); + return *this; } JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { @@ -114,12 +119,35 @@ private: }; - - struct LogicalDylibResources { typedef std::function<RuntimeDyld::SymbolInfo(const std::string&)> SymbolResolverFtor; + + typedef std::function<typename BaseLayerT::ModuleSetHandleT( + BaseLayerT&, + std::unique_ptr<Module>, + std::unique_ptr<RuntimeDyld::SymbolResolver>)> + ModuleAdderFtor; + + LogicalDylibResources() = default; + + // Explicit move constructor to make MSVC happy. + LogicalDylibResources(LogicalDylibResources &&Other) + : ExternalSymbolResolver(std::move(Other.ExternalSymbolResolver)), + MemMgr(std::move(Other.MemMgr)), + ModuleAdder(std::move(Other.ModuleAdder)) {} + + // Explicit move assignment operator to make MSVC happy. + LogicalDylibResources& operator=(LogicalDylibResources &&Other) { + ExternalSymbolResolver = std::move(Other.ExternalSymbolResolver); + MemMgr = std::move(Other.MemMgr); + ModuleAdder = std::move(Other.ModuleAdder); + return *this; + } + SymbolResolverFtor ExternalSymbolResolver; + std::unique_ptr<ResourceOwner<RuntimeDyld::MemoryManager>> MemMgr; + ModuleAdderFtor ModuleAdder; }; typedef LogicalDylib<BaseLayerT, LogicalModuleResources, @@ -157,9 +185,6 @@ public: MemoryManagerPtrT MemMgr, SymbolResolverPtrT Resolver) { - assert(MemMgr == nullptr && - "User supplied memory managers not supported with COD yet."); - LogicalDylibs.push_back(CODLogicalDylib(BaseLayer)); auto &LDResources = LogicalDylibs.back().getDylibResources(); @@ -168,6 +193,18 @@ public: return Resolver->findSymbol(Name); }; + auto &MemMgrRef = *MemMgr; + LDResources.MemMgr = + wrapOwnership<RuntimeDyld::MemoryManager>(std::move(MemMgr)); + + LDResources.ModuleAdder = + [&MemMgrRef](BaseLayerT &B, std::unique_ptr<Module> M, + std::unique_ptr<RuntimeDyld::SymbolResolver> R) { + std::vector<std::unique_ptr<Module>> Ms; + Ms.push_back(std::move(M)); + return B.addModuleSet(std::move(Ms), &MemMgrRef, std::move(R)); + }; + // Process each of the modules in this module set. for (auto &M : Ms) addLogicalModule(LogicalDylibs.back(), std::move(M)); @@ -215,9 +252,9 @@ private: auto LMH = LD.createLogicalModule(); auto &LMResources = LD.getLogicalModuleResources(LMH); - LMResources.SourceModuleOwner = wrapOwnership(std::move(SrcMPtr)); + LMResources.SourceModule = wrapOwnership<Module>(std::move(SrcMPtr)); - Module &SrcM = LMResources.SourceModuleOwner->getModule(); + Module &SrcM = LMResources.SourceModule->getResource(); // Create the GlobalValues module. const DataLayout &DL = SrcM.getDataLayout(); @@ -326,12 +363,9 @@ private: return RuntimeDyld::SymbolInfo(nullptr); }); - std::vector<std::unique_ptr<Module>> GVsMSet; - GVsMSet.push_back(std::move(GVsM)); auto GVsH = - BaseLayer.addModuleSet(std::move(GVsMSet), - llvm::make_unique<SectionMemoryManager>(), - std::move(GVsResolver)); + LD.getDylibResources().ModuleAdder(BaseLayer, std::move(GVsM), + std::move(GVsResolver)); LD.addToLogicalModule(LMH, GVsH); } @@ -348,7 +382,7 @@ private: LogicalModuleHandle LMH, Function &F) { auto &LMResources = LD.getLogicalModuleResources(LMH); - Module &SrcM = LMResources.SourceModuleOwner->getModule(); + Module &SrcM = LMResources.SourceModule->getResource(); // If F is a declaration we must already have compiled it. if (F.isDeclaration()) @@ -386,7 +420,7 @@ private: LogicalModuleHandle LMH, const PartitionT &Part) { auto &LMResources = LD.getLogicalModuleResources(LMH); - Module &SrcM = LMResources.SourceModuleOwner->getModule(); + Module &SrcM = LMResources.SourceModule->getResource(); // Create the module. std::string NewName = SrcM.getName(); @@ -445,7 +479,6 @@ private: moveFunctionBody(*F, VMap, &Materializer); // Create memory manager and symbol resolver. - auto MemMgr = llvm::make_unique<SectionMemoryManager>(); auto Resolver = createLambdaResolver( [this, &LD, LMH](const std::string &Name) { if (auto Symbol = LD.findSymbolInternally(LMH, Name)) @@ -459,10 +492,9 @@ private: Symbol.getFlags()); return RuntimeDyld::SymbolInfo(nullptr); }); - std::vector<std::unique_ptr<Module>> PartMSet; - PartMSet.push_back(std::move(M)); - return BaseLayer.addModuleSet(std::move(PartMSet), std::move(MemMgr), - std::move(Resolver)); + + return LD.getDylibResources().ModuleAdder(BaseLayer, std::move(M), + std::move(Resolver)); } BaseLayerT &BaseLayer; diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index d6ee3a846b04..e17630fa05ff 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -22,6 +22,7 @@ #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include "llvm/Support/Process.h" #include <sstream> namespace llvm { @@ -179,14 +180,15 @@ private: std::error_code EC; auto TrampolineBlock = sys::OwningMemoryBlock( - sys::Memory::allocateMappedMemory(TargetT::PageSize, nullptr, + sys::Memory::allocateMappedMemory(sys::Process::getPageSize(), nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); assert(!EC && "Failed to allocate trampoline block"); unsigned NumTrampolines = - (TargetT::PageSize - TargetT::PointerSize) / TargetT::TrampolineSize; + (sys::Process::getPageSize() - TargetT::PointerSize) / + TargetT::TrampolineSize; uint8_t *TrampolineMem = static_cast<uint8_t*>(TrampolineBlock.base()); TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), @@ -240,8 +242,8 @@ private: virtual void anchor(); }; -/// @brief IndirectStubsManager implementation for a concrete target, e.g. -/// OrcX86_64. (See OrcTargetSupport.h). +/// @brief IndirectStubsManager implementation for the host architecture, e.g. +/// OrcX86_64. (See OrcArchitectureSupport.h). template <typename TargetT> class LocalIndirectStubsManager : public IndirectStubsManager { public: diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index 2acfecfb94dc..4dc48f114883 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -108,9 +108,7 @@ private: void Finalize() override { State = Finalizing; - RTDyld->resolveRelocations(); - RTDyld->registerEHFrames(); - MemMgr->finalizeMemory(); + RTDyld->finalizeWithMemoryManagerLocking(); State = Finalized; } diff --git a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h b/include/llvm/ExecutionEngine/Orc/OrcArchitectureSupport.h index 246d3e0a9fc6..1b0488bcf00d 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcArchitectureSupport.h @@ -1,4 +1,4 @@ -//===-- OrcTargetSupport.h - Code to support specific targets --*- C++ -*-===// +//===-- OrcArchitectureSupport.h - Architecture support code ---*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,32 +7,76 @@ // //===----------------------------------------------------------------------===// // -// Target specific code for Orc, e.g. callback assembly. +// Architecture specific code for Orc, e.g. callback assembly. // -// Target classes should be part of the JIT *target* process, not the host +// Architecture classes should be part of the JIT *target* process, not the host // process (except where you're doing hosted JITing and the two are one and the // same). // //===----------------------------------------------------------------------===// -#ifndef LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H -#define LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCARCHITECTURESUPPORT_H +#define LLVM_EXECUTIONENGINE_ORC_ORCARCHITECTURESUPPORT_H #include "IndirectionUtils.h" #include "llvm/Support/Memory.h" +#include "llvm/Support/Process.h" namespace llvm { namespace orc { +/// Generic ORC Architecture support. +/// +/// This class can be substituted as the target architecure support class for +/// ORC templates that require one (e.g. IndirectStubsManagers). It does not +/// support lazy JITing however, and any attempt to use that functionality +/// will result in execution of an llvm_unreachable. +class OrcGenericArchitecture { +public: + static const unsigned PointerSize = sizeof(uintptr_t); + static const unsigned TrampolineSize = 1; + static const unsigned ResolverCodeSize = 1; + + typedef TargetAddress (*JITReentryFn)(void *CallbackMgr, void *TrampolineId); + + static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, + void *CallbackMgr) { + llvm_unreachable("writeResolverCode is not supported by the generic host " + "support class"); + } + + static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { + llvm_unreachable("writeTrampolines is not supported by the generic host " + "support class"); + } + + class IndirectStubsInfo { + public: + const static unsigned StubSize = 1; + unsigned getNumStubs() const { llvm_unreachable("Not supported"); } + void *getStub(unsigned Idx) const { llvm_unreachable("Not supported"); } + void **getPtr(unsigned Idx) const { llvm_unreachable("Not supported"); } + }; + + static std::error_code emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + llvm_unreachable("emitIndirectStubsBlock is not supported by the generic " + "host support class"); + } +}; + +/// @brief X86_64 support. +/// +/// X86_64 supports lazy JITing. class OrcX86_64 { public: - static const unsigned PageSize = 4096; static const unsigned PointerSize = 8; static const unsigned TrampolineSize = 8; static const unsigned ResolverCodeSize = 0x78; - typedef TargetAddress (*JITReentryFn)(void *CallbackMgr, - void *TrampolineId); + typedef TargetAddress (*JITReentryFn)(void *CallbackMgr, void *TrampolineId); /// @brief Write the resolver code into the given memory. The user is be /// responsible for allocating the memory and setting permissions. @@ -49,16 +93,16 @@ public: /// makeIndirectStubsBlock function. class IndirectStubsInfo { friend class OrcX86_64; + public: const static unsigned StubSize = 8; - const static unsigned PtrSize = 8; IndirectStubsInfo() : NumStubs(0) {} IndirectStubsInfo(IndirectStubsInfo &&Other) : NumStubs(Other.NumStubs), StubsMem(std::move(Other.StubsMem)) { Other.NumStubs = 0; } - IndirectStubsInfo& operator=(IndirectStubsInfo &&Other) { + IndirectStubsInfo &operator=(IndirectStubsInfo &&Other) { NumStubs = Other.NumStubs; Other.NumStubs = 0; StubsMem = std::move(Other.StubsMem); @@ -70,17 +114,18 @@ public: /// @brief Get a pointer to the stub at the given index, which must be in /// the range 0 .. getNumStubs() - 1. - void* getStub(unsigned Idx) const { - return static_cast<uint64_t*>(StubsMem.base()) + Idx; + void *getStub(unsigned Idx) const { + return static_cast<uint64_t *>(StubsMem.base()) + Idx; } /// @brief Get a pointer to the implementation-pointer at the given index, /// which must be in the range 0 .. getNumStubs() - 1. - void** getPtr(unsigned Idx) const { + void **getPtr(unsigned Idx) const { char *PtrsBase = - static_cast<char*>(StubsMem.base()) + NumStubs * StubSize; - return reinterpret_cast<void**>(PtrsBase) + Idx; + static_cast<char *>(StubsMem.base()) + NumStubs * StubSize; + return reinterpret_cast<void **>(PtrsBase) + Idx; } + private: unsigned NumStubs; sys::OwningMemoryBlock StubsMem; @@ -100,4 +145,4 @@ public: } // End namespace orc. } // End namespace llvm. -#endif // LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H +#endif // LLVM_EXECUTIONENGINE_ORC_ORCARCHITECTURESUPPORT_H diff --git a/include/llvm/ExecutionEngine/Orc/OrcError.h b/include/llvm/ExecutionEngine/Orc/OrcError.h new file mode 100644 index 000000000000..48f35d6b39be --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -0,0 +1,37 @@ +//===------ OrcError.h - Reject symbol lookup requests ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Define an error category, error codes, and helper utilities for Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCERROR_H +#define LLVM_EXECUTIONENGINE_ORC_ORCERROR_H + +#include <system_error> + +namespace llvm { +namespace orc { + +enum class OrcErrorCode : int { + // RPC Errors + RemoteAllocatorDoesNotExist = 1, + RemoteAllocatorIdAlreadyInUse, + RemoteMProtectAddrUnrecognized, + RemoteIndirectStubsOwnerDoesNotExist, + RemoteIndirectStubsOwnerIdAlreadyInUse, + UnexpectedRPCCall +}; + +std::error_code orcError(OrcErrorCode ErrCode); + +} // End namespace orc. +} // End namespace llvm. + +#endif // LLVM_EXECUTIONENGINE_ORC_ORCERROR_H diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h new file mode 100644 index 000000000000..d7640b8e8b5f --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -0,0 +1,784 @@ +//===---- OrcRemoteTargetClient.h - Orc Remote-target Client ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the OrcRemoteTargetClient class and helpers. This class +// can be used to communicate over an RPCChannel with an OrcRemoteTargetServer +// instance to support remote-JITing. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETCLIENT_H +#define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETCLIENT_H + +#include "IndirectionUtils.h" +#include "OrcRemoteTargetRPCAPI.h" +#include <system_error> + +#define DEBUG_TYPE "orc-remote" + +namespace llvm { +namespace orc { +namespace remote { + +/// This class provides utilities (including memory manager, indirect stubs +/// manager, and compile callback manager types) that support remote JITing +/// in ORC. +/// +/// Each of the utility classes talks to a JIT server (an instance of the +/// OrcRemoteTargetServer class) via an RPC system (see RPCUtils.h) to carry out +/// its actions. +template <typename ChannelT> +class OrcRemoteTargetClient : public OrcRemoteTargetRPCAPI { +public: + /// Remote memory manager. + class RCMemoryManager : public RuntimeDyld::MemoryManager { + public: + RCMemoryManager(OrcRemoteTargetClient &Client, ResourceIdMgr::ResourceId Id) + : Client(Client), Id(Id) { + DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); + } + + RCMemoryManager(RCMemoryManager &&Other) + : Client(std::move(Other.Client)), Id(std::move(Other.Id)), + Unmapped(std::move(Other.Unmapped)), + Unfinalized(std::move(Other.Unfinalized)) {} + + RCMemoryManager operator=(RCMemoryManager &&Other) { + Client = std::move(Other.Client); + Id = std::move(Other.Id); + Unmapped = std::move(Other.Unmapped); + Unfinalized = std::move(Other.Unfinalized); + return *this; + } + + ~RCMemoryManager() { + Client.destroyRemoteAllocator(Id); + DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); + } + + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override { + Unmapped.back().CodeAllocs.emplace_back(Size, Alignment); + uint8_t *Alloc = reinterpret_cast<uint8_t *>( + Unmapped.back().CodeAllocs.back().getLocalAddress()); + DEBUG(dbgs() << "Allocator " << Id << " allocated code for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); + return Alloc; + } + + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool IsReadOnly) override { + if (IsReadOnly) { + Unmapped.back().RODataAllocs.emplace_back(Size, Alignment); + uint8_t *Alloc = reinterpret_cast<uint8_t *>( + Unmapped.back().RODataAllocs.back().getLocalAddress()); + DEBUG(dbgs() << "Allocator " << Id << " allocated ro-data for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); + return Alloc; + } // else... + + Unmapped.back().RWDataAllocs.emplace_back(Size, Alignment); + uint8_t *Alloc = reinterpret_cast<uint8_t *>( + Unmapped.back().RWDataAllocs.back().getLocalAddress()); + DEBUG(dbgs() << "Allocator " << Id << " allocated rw-data for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); + return Alloc; + } + + void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override { + Unmapped.push_back(ObjectAllocs()); + + DEBUG(dbgs() << "Allocator " << Id << " reserved:\n"); + + if (CodeSize != 0) { + std::error_code EC = Client.reserveMem(Unmapped.back().RemoteCodeAddr, + Id, CodeSize, CodeAlign); + // FIXME; Add error to poll. + assert(!EC && "Failed reserving remote memory."); + (void)EC; + DEBUG(dbgs() << " code: " + << format("0x%016x", Unmapped.back().RemoteCodeAddr) + << " (" << CodeSize << " bytes, alignment " << CodeAlign + << ")\n"); + } + + if (RODataSize != 0) { + std::error_code EC = Client.reserveMem(Unmapped.back().RemoteRODataAddr, + Id, RODataSize, RODataAlign); + // FIXME; Add error to poll. + assert(!EC && "Failed reserving remote memory."); + (void)EC; + DEBUG(dbgs() << " ro-data: " + << format("0x%016x", Unmapped.back().RemoteRODataAddr) + << " (" << RODataSize << " bytes, alignment " + << RODataAlign << ")\n"); + } + + if (RWDataSize != 0) { + std::error_code EC = Client.reserveMem(Unmapped.back().RemoteRWDataAddr, + Id, RWDataSize, RWDataAlign); + // FIXME; Add error to poll. + assert(!EC && "Failed reserving remote memory."); + (void)EC; + DEBUG(dbgs() << " rw-data: " + << format("0x%016x", Unmapped.back().RemoteRWDataAddr) + << " (" << RWDataSize << " bytes, alignment " + << RWDataAlign << ")\n"); + } + } + + bool needsToReserveAllocationSpace() override { return true; } + + void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, + size_t Size) override {} + + void deregisterEHFrames(uint8_t *addr, uint64_t LoadAddr, + size_t Size) override {} + + void notifyObjectLoaded(RuntimeDyld &Dyld, + const object::ObjectFile &Obj) override { + DEBUG(dbgs() << "Allocator " << Id << " applied mappings:\n"); + for (auto &ObjAllocs : Unmapped) { + { + TargetAddress NextCodeAddr = ObjAllocs.RemoteCodeAddr; + for (auto &Alloc : ObjAllocs.CodeAllocs) { + NextCodeAddr = RoundUpToAlignment(NextCodeAddr, Alloc.getAlign()); + Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextCodeAddr); + DEBUG(dbgs() << " code: " + << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " << format("0x%016x", NextCodeAddr) << "\n"); + Alloc.setRemoteAddress(NextCodeAddr); + NextCodeAddr += Alloc.getSize(); + } + } + { + TargetAddress NextRODataAddr = ObjAllocs.RemoteRODataAddr; + for (auto &Alloc : ObjAllocs.RODataAllocs) { + NextRODataAddr = + RoundUpToAlignment(NextRODataAddr, Alloc.getAlign()); + Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextRODataAddr); + DEBUG(dbgs() << " ro-data: " + << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " << format("0x%016x", NextRODataAddr) + << "\n"); + Alloc.setRemoteAddress(NextRODataAddr); + NextRODataAddr += Alloc.getSize(); + } + } + { + TargetAddress NextRWDataAddr = ObjAllocs.RemoteRWDataAddr; + for (auto &Alloc : ObjAllocs.RWDataAllocs) { + NextRWDataAddr = + RoundUpToAlignment(NextRWDataAddr, Alloc.getAlign()); + Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextRWDataAddr); + DEBUG(dbgs() << " rw-data: " + << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " << format("0x%016x", NextRWDataAddr) + << "\n"); + Alloc.setRemoteAddress(NextRWDataAddr); + NextRWDataAddr += Alloc.getSize(); + } + } + Unfinalized.push_back(std::move(ObjAllocs)); + } + Unmapped.clear(); + } + + bool finalizeMemory(std::string *ErrMsg = nullptr) override { + DEBUG(dbgs() << "Allocator " << Id << " finalizing:\n"); + + for (auto &ObjAllocs : Unfinalized) { + + for (auto &Alloc : ObjAllocs.CodeAllocs) { + DEBUG(dbgs() << " copying code: " + << static_cast<void *>(Alloc.getLocalAddress()) << " -> " + << format("0x%016x", Alloc.getRemoteAddress()) << " (" + << Alloc.getSize() << " bytes)\n"); + Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), + Alloc.getSize()); + } + + if (ObjAllocs.RemoteCodeAddr) { + DEBUG(dbgs() << " setting R-X permissions on code block: " + << format("0x%016x", ObjAllocs.RemoteCodeAddr) << "\n"); + Client.setProtections(Id, ObjAllocs.RemoteCodeAddr, + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + } + + for (auto &Alloc : ObjAllocs.RODataAllocs) { + DEBUG(dbgs() << " copying ro-data: " + << static_cast<void *>(Alloc.getLocalAddress()) << " -> " + << format("0x%016x", Alloc.getRemoteAddress()) << " (" + << Alloc.getSize() << " bytes)\n"); + Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), + Alloc.getSize()); + } + + if (ObjAllocs.RemoteRODataAddr) { + DEBUG(dbgs() << " setting R-- permissions on ro-data block: " + << format("0x%016x", ObjAllocs.RemoteRODataAddr) + << "\n"); + Client.setProtections(Id, ObjAllocs.RemoteRODataAddr, + sys::Memory::MF_READ); + } + + for (auto &Alloc : ObjAllocs.RWDataAllocs) { + DEBUG(dbgs() << " copying rw-data: " + << static_cast<void *>(Alloc.getLocalAddress()) << " -> " + << format("0x%016x", Alloc.getRemoteAddress()) << " (" + << Alloc.getSize() << " bytes)\n"); + Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), + Alloc.getSize()); + } + + if (ObjAllocs.RemoteRWDataAddr) { + DEBUG(dbgs() << " setting RW- permissions on rw-data block: " + << format("0x%016x", ObjAllocs.RemoteRWDataAddr) + << "\n"); + Client.setProtections(Id, ObjAllocs.RemoteRWDataAddr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + } + } + Unfinalized.clear(); + + return false; + } + + private: + class Alloc { + public: + Alloc(uint64_t Size, unsigned Align) + : Size(Size), Align(Align), Contents(new char[Size + Align - 1]), + RemoteAddr(0) {} + + Alloc(Alloc &&Other) + : Size(std::move(Other.Size)), Align(std::move(Other.Align)), + Contents(std::move(Other.Contents)), + RemoteAddr(std::move(Other.RemoteAddr)) {} + + Alloc &operator=(Alloc &&Other) { + Size = std::move(Other.Size); + Align = std::move(Other.Align); + Contents = std::move(Other.Contents); + RemoteAddr = std::move(Other.RemoteAddr); + return *this; + } + + uint64_t getSize() const { return Size; } + + unsigned getAlign() const { return Align; } + + char *getLocalAddress() const { + uintptr_t LocalAddr = reinterpret_cast<uintptr_t>(Contents.get()); + LocalAddr = RoundUpToAlignment(LocalAddr, Align); + return reinterpret_cast<char *>(LocalAddr); + } + + void setRemoteAddress(TargetAddress RemoteAddr) { + this->RemoteAddr = RemoteAddr; + } + + TargetAddress getRemoteAddress() const { return RemoteAddr; } + + private: + uint64_t Size; + unsigned Align; + std::unique_ptr<char[]> Contents; + TargetAddress RemoteAddr; + }; + + struct ObjectAllocs { + ObjectAllocs() + : RemoteCodeAddr(0), RemoteRODataAddr(0), RemoteRWDataAddr(0) {} + + ObjectAllocs(ObjectAllocs &&Other) + : RemoteCodeAddr(std::move(Other.RemoteCodeAddr)), + RemoteRODataAddr(std::move(Other.RemoteRODataAddr)), + RemoteRWDataAddr(std::move(Other.RemoteRWDataAddr)), + CodeAllocs(std::move(Other.CodeAllocs)), + RODataAllocs(std::move(Other.RODataAllocs)), + RWDataAllocs(std::move(Other.RWDataAllocs)) {} + + ObjectAllocs &operator=(ObjectAllocs &&Other) { + RemoteCodeAddr = std::move(Other.RemoteCodeAddr); + RemoteRODataAddr = std::move(Other.RemoteRODataAddr); + RemoteRWDataAddr = std::move(Other.RemoteRWDataAddr); + CodeAllocs = std::move(Other.CodeAllocs); + RODataAllocs = std::move(Other.RODataAllocs); + RWDataAllocs = std::move(Other.RWDataAllocs); + return *this; + } + + TargetAddress RemoteCodeAddr; + TargetAddress RemoteRODataAddr; + TargetAddress RemoteRWDataAddr; + std::vector<Alloc> CodeAllocs, RODataAllocs, RWDataAllocs; + }; + + OrcRemoteTargetClient &Client; + ResourceIdMgr::ResourceId Id; + std::vector<ObjectAllocs> Unmapped; + std::vector<ObjectAllocs> Unfinalized; + }; + + /// Remote indirect stubs manager. + class RCIndirectStubsManager : public IndirectStubsManager { + public: + RCIndirectStubsManager(OrcRemoteTargetClient &Remote, + ResourceIdMgr::ResourceId Id) + : Remote(Remote), Id(Id) {} + + ~RCIndirectStubsManager() { Remote.destroyIndirectStubsManager(Id); } + + std::error_code createStub(StringRef StubName, TargetAddress StubAddr, + JITSymbolFlags StubFlags) override { + if (auto EC = reserveStubs(1)) + return EC; + + return createStubInternal(StubName, StubAddr, StubFlags); + } + + std::error_code createStubs(const StubInitsMap &StubInits) override { + if (auto EC = reserveStubs(StubInits.size())) + return EC; + + for (auto &Entry : StubInits) + if (auto EC = createStubInternal(Entry.first(), Entry.second.first, + Entry.second.second)) + return EC; + + return std::error_code(); + } + + JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + auto Flags = I->second.second; + auto StubSymbol = JITSymbol(getStubAddr(Key), Flags); + if (ExportedStubsOnly && !StubSymbol.isExported()) + return nullptr; + return StubSymbol; + } + + JITSymbol findPointer(StringRef Name) override { + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + auto Flags = I->second.second; + return JITSymbol(getPtrAddr(Key), Flags); + } + + std::error_code updatePointer(StringRef Name, + TargetAddress NewAddr) override { + auto I = StubIndexes.find(Name); + assert(I != StubIndexes.end() && "No stub pointer for symbol"); + auto Key = I->second.first; + return Remote.writePointer(getPtrAddr(Key), NewAddr); + } + + private: + struct RemoteIndirectStubsInfo { + RemoteIndirectStubsInfo(TargetAddress StubBase, TargetAddress PtrBase, + unsigned NumStubs) + : StubBase(StubBase), PtrBase(PtrBase), NumStubs(NumStubs) {} + TargetAddress StubBase; + TargetAddress PtrBase; + unsigned NumStubs; + }; + + OrcRemoteTargetClient &Remote; + ResourceIdMgr::ResourceId Id; + std::vector<RemoteIndirectStubsInfo> RemoteIndirectStubsInfos; + typedef std::pair<uint16_t, uint16_t> StubKey; + std::vector<StubKey> FreeStubs; + StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; + + std::error_code reserveStubs(unsigned NumStubs) { + if (NumStubs <= FreeStubs.size()) + return std::error_code(); + + unsigned NewStubsRequired = NumStubs - FreeStubs.size(); + TargetAddress StubBase; + TargetAddress PtrBase; + unsigned NumStubsEmitted; + + Remote.emitIndirectStubs(StubBase, PtrBase, NumStubsEmitted, Id, + NewStubsRequired); + + unsigned NewBlockId = RemoteIndirectStubsInfos.size(); + RemoteIndirectStubsInfos.push_back( + RemoteIndirectStubsInfo(StubBase, PtrBase, NumStubsEmitted)); + + for (unsigned I = 0; I < NumStubsEmitted; ++I) + FreeStubs.push_back(std::make_pair(NewBlockId, I)); + + return std::error_code(); + } + + std::error_code createStubInternal(StringRef StubName, + TargetAddress InitAddr, + JITSymbolFlags StubFlags) { + auto Key = FreeStubs.back(); + FreeStubs.pop_back(); + StubIndexes[StubName] = std::make_pair(Key, StubFlags); + return Remote.writePointer(getPtrAddr(Key), InitAddr); + } + + TargetAddress getStubAddr(StubKey K) { + assert(RemoteIndirectStubsInfos[K.first].StubBase != 0 && + "Missing stub address"); + return RemoteIndirectStubsInfos[K.first].StubBase + + K.second * Remote.getIndirectStubSize(); + } + + TargetAddress getPtrAddr(StubKey K) { + assert(RemoteIndirectStubsInfos[K.first].PtrBase != 0 && + "Missing pointer address"); + return RemoteIndirectStubsInfos[K.first].PtrBase + + K.second * Remote.getPointerSize(); + } + }; + + /// Remote compile callback manager. + class RCCompileCallbackManager : public JITCompileCallbackManager { + public: + RCCompileCallbackManager(TargetAddress ErrorHandlerAddress, + OrcRemoteTargetClient &Remote) + : JITCompileCallbackManager(ErrorHandlerAddress), Remote(Remote) { + assert(!Remote.CompileCallback && "Compile callback already set"); + Remote.CompileCallback = [this](TargetAddress TrampolineAddr) { + return executeCompileCallback(TrampolineAddr); + }; + Remote.emitResolverBlock(); + } + + private: + void grow() { + TargetAddress BlockAddr = 0; + uint32_t NumTrampolines = 0; + auto EC = Remote.emitTrampolineBlock(BlockAddr, NumTrampolines); + assert(!EC && "Failed to create trampolines"); + + uint32_t TrampolineSize = Remote.getTrampolineSize(); + for (unsigned I = 0; I < NumTrampolines; ++I) + this->AvailableTrampolines.push_back(BlockAddr + (I * TrampolineSize)); + } + + OrcRemoteTargetClient &Remote; + }; + + /// Create an OrcRemoteTargetClient. + /// Channel is the ChannelT instance to communicate on. It is assumed that + /// the channel is ready to be read from and written to. + static ErrorOr<OrcRemoteTargetClient> Create(ChannelT &Channel) { + std::error_code EC; + OrcRemoteTargetClient H(Channel, EC); + if (EC) + return EC; + return H; + } + + /// Call the int(void) function at the given address in the target and return + /// its result. + std::error_code callIntVoid(int &Result, TargetAddress Addr) { + DEBUG(dbgs() << "Calling int(*)(void) " << format("0x%016x", Addr) << "\n"); + + if (auto EC = call<CallIntVoid>(Channel, Addr)) + return EC; + + unsigned NextProcId; + if (auto EC = listenForCompileRequests(NextProcId)) + return EC; + + if (NextProcId != CallIntVoidResponseId) + return orcError(OrcErrorCode::UnexpectedRPCCall); + + return handle<CallIntVoidResponse>(Channel, [&](int R) { + Result = R; + DEBUG(dbgs() << "Result: " << R << "\n"); + return std::error_code(); + }); + } + + /// Call the int(int, char*[]) function at the given address in the target and + /// return its result. + std::error_code callMain(int &Result, TargetAddress Addr, + const std::vector<std::string> &Args) { + DEBUG(dbgs() << "Calling int(*)(int, char*[]) " << format("0x%016x", Addr) + << "\n"); + + if (auto EC = call<CallMain>(Channel, Addr, Args)) + return EC; + + unsigned NextProcId; + if (auto EC = listenForCompileRequests(NextProcId)) + return EC; + + if (NextProcId != CallMainResponseId) + return orcError(OrcErrorCode::UnexpectedRPCCall); + + return handle<CallMainResponse>(Channel, [&](int R) { + Result = R; + DEBUG(dbgs() << "Result: " << R << "\n"); + return std::error_code(); + }); + } + + /// Call the void() function at the given address in the target and wait for + /// it to finish. + std::error_code callVoidVoid(TargetAddress Addr) { + DEBUG(dbgs() << "Calling void(*)(void) " << format("0x%016x", Addr) + << "\n"); + + if (auto EC = call<CallVoidVoid>(Channel, Addr)) + return EC; + + unsigned NextProcId; + if (auto EC = listenForCompileRequests(NextProcId)) + return EC; + + if (NextProcId != CallVoidVoidResponseId) + return orcError(OrcErrorCode::UnexpectedRPCCall); + + return handle<CallVoidVoidResponse>(Channel, doNothing); + } + + /// Create an RCMemoryManager which will allocate its memory on the remote + /// target. + std::error_code + createRemoteMemoryManager(std::unique_ptr<RCMemoryManager> &MM) { + assert(!MM && "MemoryManager should be null before creation."); + + auto Id = AllocatorIds.getNext(); + if (auto EC = call<CreateRemoteAllocator>(Channel, Id)) + return EC; + MM = llvm::make_unique<RCMemoryManager>(*this, Id); + return std::error_code(); + } + + /// Create an RCIndirectStubsManager that will allocate stubs on the remote + /// target. + std::error_code + createIndirectStubsManager(std::unique_ptr<RCIndirectStubsManager> &I) { + assert(!I && "Indirect stubs manager should be null before creation."); + auto Id = IndirectStubOwnerIds.getNext(); + if (auto EC = call<CreateIndirectStubsOwner>(Channel, Id)) + return EC; + I = llvm::make_unique<RCIndirectStubsManager>(*this, Id); + return std::error_code(); + } + + /// Search for symbols in the remote process. Note: This should be used by + /// symbol resolvers *after* they've searched the local symbol table in the + /// JIT stack. + std::error_code getSymbolAddress(TargetAddress &Addr, StringRef Name) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + // Request remote symbol address. + if (auto EC = call<GetSymbolAddress>(Channel, Name)) + return EC; + + return expect<GetSymbolAddressResponse>(Channel, [&](TargetAddress &A) { + Addr = A; + DEBUG(dbgs() << "Remote address lookup " << Name << " = " + << format("0x%016x", Addr) << "\n"); + return std::error_code(); + }); + } + + /// Get the triple for the remote target. + const std::string &getTargetTriple() const { return RemoteTargetTriple; } + + std::error_code terminateSession() { return call<TerminateSession>(Channel); } + +private: + OrcRemoteTargetClient(ChannelT &Channel, std::error_code &EC) + : Channel(Channel), RemotePointerSize(0), RemotePageSize(0), + RemoteTrampolineSize(0), RemoteIndirectStubSize(0) { + if ((EC = call<GetRemoteInfo>(Channel))) + return; + + EC = expect<GetRemoteInfoResponse>( + Channel, readArgs(RemoteTargetTriple, RemotePointerSize, RemotePageSize, + RemoteTrampolineSize, RemoteIndirectStubSize)); + } + + void destroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { + if (auto EC = call<DestroyRemoteAllocator>(Channel, Id)) { + // FIXME: This will be triggered by a removeModuleSet call: Propagate + // error return up through that. + llvm_unreachable("Failed to destroy remote allocator."); + AllocatorIds.release(Id); + } + } + + std::error_code destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { + IndirectStubOwnerIds.release(Id); + return call<DestroyIndirectStubsOwner>(Channel, Id); + } + + std::error_code emitIndirectStubs(TargetAddress &StubBase, + TargetAddress &PtrBase, + uint32_t &NumStubsEmitted, + ResourceIdMgr::ResourceId Id, + uint32_t NumStubsRequired) { + if (auto EC = call<EmitIndirectStubs>(Channel, Id, NumStubsRequired)) + return EC; + + return expect<EmitIndirectStubsResponse>( + Channel, readArgs(StubBase, PtrBase, NumStubsEmitted)); + } + + std::error_code emitResolverBlock() { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + return call<EmitResolverBlock>(Channel); + } + + std::error_code emitTrampolineBlock(TargetAddress &BlockAddr, + uint32_t &NumTrampolines) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + if (auto EC = call<EmitTrampolineBlock>(Channel)) + return EC; + + return expect<EmitTrampolineBlockResponse>( + Channel, [&](TargetAddress BAddr, uint32_t NTrampolines) { + BlockAddr = BAddr; + NumTrampolines = NTrampolines; + return std::error_code(); + }); + } + + uint32_t getIndirectStubSize() const { return RemoteIndirectStubSize; } + uint32_t getPageSize() const { return RemotePageSize; } + uint32_t getPointerSize() const { return RemotePointerSize; } + + uint32_t getTrampolineSize() const { return RemoteTrampolineSize; } + + std::error_code listenForCompileRequests(uint32_t &NextId) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + if (auto EC = getNextProcId(Channel, NextId)) + return EC; + + while (NextId == RequestCompileId) { + TargetAddress TrampolineAddr = 0; + if (auto EC = handle<RequestCompile>(Channel, readArgs(TrampolineAddr))) + return EC; + + TargetAddress ImplAddr = CompileCallback(TrampolineAddr); + if (auto EC = call<RequestCompileResponse>(Channel, ImplAddr)) + return EC; + + if (auto EC = getNextProcId(Channel, NextId)) + return EC; + } + + return std::error_code(); + } + + std::error_code readMem(char *Dst, TargetAddress Src, uint64_t Size) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + if (auto EC = call<ReadMem>(Channel, Src, Size)) + return EC; + + if (auto EC = expect<ReadMemResponse>( + Channel, [&]() { return Channel.readBytes(Dst, Size); })) + return EC; + + return std::error_code(); + } + + std::error_code reserveMem(TargetAddress &RemoteAddr, + ResourceIdMgr::ResourceId Id, uint64_t Size, + uint32_t Align) { + + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + if (std::error_code EC = call<ReserveMem>(Channel, Id, Size, Align)) + return EC; + + return expect<ReserveMemResponse>(Channel, readArgs(RemoteAddr)); + } + + std::error_code setProtections(ResourceIdMgr::ResourceId Id, + TargetAddress RemoteSegAddr, + unsigned ProtFlags) { + return call<SetProtections>(Channel, Id, RemoteSegAddr, ProtFlags); + } + + std::error_code writeMem(TargetAddress Addr, const char *Src, uint64_t Size) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + // Make the send call. + if (auto EC = call<WriteMem>(Channel, Addr, Size)) + return EC; + + // Follow this up with the section contents. + if (auto EC = Channel.appendBytes(Src, Size)) + return EC; + + return Channel.send(); + } + + std::error_code writePointer(TargetAddress Addr, TargetAddress PtrVal) { + // Check for an 'out-of-band' error, e.g. from an MM destructor. + if (ExistingError) + return ExistingError; + + return call<WritePtr>(Channel, Addr, PtrVal); + } + + static std::error_code doNothing() { return std::error_code(); } + + ChannelT &Channel; + std::error_code ExistingError; + std::string RemoteTargetTriple; + uint32_t RemotePointerSize; + uint32_t RemotePageSize; + uint32_t RemoteTrampolineSize; + uint32_t RemoteIndirectStubSize; + ResourceIdMgr AllocatorIds, IndirectStubOwnerIds; + std::function<TargetAddress(TargetAddress)> CompileCallback; +}; + +} // end namespace remote +} // end namespace orc +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h new file mode 100644 index 000000000000..96dc24251026 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -0,0 +1,185 @@ +//===--- OrcRemoteTargetRPCAPI.h - Orc Remote-target RPC API ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the Orc remote-target RPC API. It should not be used +// directly, but is used by the RemoteTargetClient and RemoteTargetServer +// classes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETRPCAPI_H +#define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETRPCAPI_H + +#include "JITSymbol.h" +#include "RPCChannel.h" +#include "RPCUtils.h" + +namespace llvm { +namespace orc { +namespace remote { + +class OrcRemoteTargetRPCAPI : public RPC<RPCChannel> { +protected: + class ResourceIdMgr { + public: + typedef uint64_t ResourceId; + ResourceIdMgr() : NextId(0) {} + ResourceId getNext() { + if (!FreeIds.empty()) { + ResourceId I = FreeIds.back(); + FreeIds.pop_back(); + return I; + } + return NextId++; + } + void release(ResourceId I) { FreeIds.push_back(I); } + + private: + ResourceId NextId; + std::vector<ResourceId> FreeIds; + }; + +public: + enum JITProcId : uint32_t { + InvalidId = 0, + CallIntVoidId, + CallIntVoidResponseId, + CallMainId, + CallMainResponseId, + CallVoidVoidId, + CallVoidVoidResponseId, + CreateRemoteAllocatorId, + CreateIndirectStubsOwnerId, + DestroyRemoteAllocatorId, + DestroyIndirectStubsOwnerId, + EmitIndirectStubsId, + EmitIndirectStubsResponseId, + EmitResolverBlockId, + EmitTrampolineBlockId, + EmitTrampolineBlockResponseId, + GetSymbolAddressId, + GetSymbolAddressResponseId, + GetRemoteInfoId, + GetRemoteInfoResponseId, + ReadMemId, + ReadMemResponseId, + ReserveMemId, + ReserveMemResponseId, + RequestCompileId, + RequestCompileResponseId, + SetProtectionsId, + TerminateSessionId, + WriteMemId, + WritePtrId + }; + + static const char *getJITProcIdName(JITProcId Id); + + typedef Procedure<CallIntVoidId, TargetAddress /* FnAddr */> CallIntVoid; + + typedef Procedure<CallIntVoidResponseId, int /* Result */> + CallIntVoidResponse; + + typedef Procedure<CallMainId, TargetAddress /* FnAddr */, + std::vector<std::string> /* Args */> + CallMain; + + typedef Procedure<CallMainResponseId, int /* Result */> CallMainResponse; + + typedef Procedure<CallVoidVoidId, TargetAddress /* FnAddr */> CallVoidVoid; + + typedef Procedure<CallVoidVoidResponseId> CallVoidVoidResponse; + + typedef Procedure<CreateRemoteAllocatorId, + ResourceIdMgr::ResourceId /* Allocator ID */> + CreateRemoteAllocator; + + typedef Procedure<CreateIndirectStubsOwnerId, + ResourceIdMgr::ResourceId /* StubsOwner ID */> + CreateIndirectStubsOwner; + + typedef Procedure<DestroyRemoteAllocatorId, + ResourceIdMgr::ResourceId /* Allocator ID */> + DestroyRemoteAllocator; + + typedef Procedure<DestroyIndirectStubsOwnerId, + ResourceIdMgr::ResourceId /* StubsOwner ID */> + DestroyIndirectStubsOwner; + + typedef Procedure<EmitIndirectStubsId, + ResourceIdMgr::ResourceId /* StubsOwner ID */, + uint32_t /* NumStubsRequired */> + EmitIndirectStubs; + + typedef Procedure< + EmitIndirectStubsResponseId, TargetAddress /* StubsBaseAddr */, + TargetAddress /* PtrsBaseAddr */, uint32_t /* NumStubsEmitted */> + EmitIndirectStubsResponse; + + typedef Procedure<EmitResolverBlockId> EmitResolverBlock; + + typedef Procedure<EmitTrampolineBlockId> EmitTrampolineBlock; + + typedef Procedure<EmitTrampolineBlockResponseId, + TargetAddress /* BlockAddr */, + uint32_t /* NumTrampolines */> + EmitTrampolineBlockResponse; + + typedef Procedure<GetSymbolAddressId, std::string /*SymbolName*/> + GetSymbolAddress; + + typedef Procedure<GetSymbolAddressResponseId, uint64_t /* SymbolAddr */> + GetSymbolAddressResponse; + + typedef Procedure<GetRemoteInfoId> GetRemoteInfo; + + typedef Procedure<GetRemoteInfoResponseId, std::string /* Triple */, + uint32_t /* PointerSize */, uint32_t /* PageSize */, + uint32_t /* TrampolineSize */, + uint32_t /* IndirectStubSize */> + GetRemoteInfoResponse; + + typedef Procedure<ReadMemId, TargetAddress /* Src */, uint64_t /* Size */> + ReadMem; + + typedef Procedure<ReadMemResponseId> ReadMemResponse; + + typedef Procedure<ReserveMemId, ResourceIdMgr::ResourceId /* Id */, + uint64_t /* Size */, uint32_t /* Align */> + ReserveMem; + + typedef Procedure<ReserveMemResponseId, TargetAddress /* Addr */> + ReserveMemResponse; + + typedef Procedure<RequestCompileId, TargetAddress /* TrampolineAddr */> + RequestCompile; + + typedef Procedure<RequestCompileResponseId, TargetAddress /* ImplAddr */> + RequestCompileResponse; + + typedef Procedure<SetProtectionsId, ResourceIdMgr::ResourceId /* Id */, + TargetAddress /* Dst */, uint32_t /* ProtFlags */> + SetProtections; + + typedef Procedure<TerminateSessionId> TerminateSession; + + typedef Procedure<WriteMemId, TargetAddress /* Dst */, uint64_t /* Size */ + /* Data should follow */> + WriteMem; + + typedef Procedure<WritePtrId, TargetAddress /* Dst */, + TargetAddress /* Val */> + WritePtr; +}; + +} // end namespace remote +} // end namespace orc +} // end namespace llvm + +#endif diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h new file mode 100644 index 000000000000..5247661e49ce --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -0,0 +1,432 @@ +//===---- OrcRemoteTargetServer.h - Orc Remote-target Server ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the OrcRemoteTargetServer class. It can be used to build a +// JIT server that can execute code sent from an OrcRemoteTargetClient. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETSERVER_H +#define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETSERVER_H + +#include "OrcRemoteTargetRPCAPI.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" +#include <map> + +#define DEBUG_TYPE "orc-remote" + +namespace llvm { +namespace orc { +namespace remote { + +template <typename ChannelT, typename TargetT> +class OrcRemoteTargetServer : public OrcRemoteTargetRPCAPI { +public: + typedef std::function<TargetAddress(const std::string &Name)> + SymbolLookupFtor; + + OrcRemoteTargetServer(ChannelT &Channel, SymbolLookupFtor SymbolLookup) + : Channel(Channel), SymbolLookup(std::move(SymbolLookup)) {} + + std::error_code getNextProcId(JITProcId &Id) { + return deserialize(Channel, Id); + } + + std::error_code handleKnownProcedure(JITProcId Id) { + typedef OrcRemoteTargetServer ThisT; + + DEBUG(dbgs() << "Handling known proc: " << getJITProcIdName(Id) << "\n"); + + switch (Id) { + case CallIntVoidId: + return handle<CallIntVoid>(Channel, *this, &ThisT::handleCallIntVoid); + case CallMainId: + return handle<CallMain>(Channel, *this, &ThisT::handleCallMain); + case CallVoidVoidId: + return handle<CallVoidVoid>(Channel, *this, &ThisT::handleCallVoidVoid); + case CreateRemoteAllocatorId: + return handle<CreateRemoteAllocator>(Channel, *this, + &ThisT::handleCreateRemoteAllocator); + case CreateIndirectStubsOwnerId: + return handle<CreateIndirectStubsOwner>( + Channel, *this, &ThisT::handleCreateIndirectStubsOwner); + case DestroyRemoteAllocatorId: + return handle<DestroyRemoteAllocator>( + Channel, *this, &ThisT::handleDestroyRemoteAllocator); + case DestroyIndirectStubsOwnerId: + return handle<DestroyIndirectStubsOwner>( + Channel, *this, &ThisT::handleDestroyIndirectStubsOwner); + case EmitIndirectStubsId: + return handle<EmitIndirectStubs>(Channel, *this, + &ThisT::handleEmitIndirectStubs); + case EmitResolverBlockId: + return handle<EmitResolverBlock>(Channel, *this, + &ThisT::handleEmitResolverBlock); + case EmitTrampolineBlockId: + return handle<EmitTrampolineBlock>(Channel, *this, + &ThisT::handleEmitTrampolineBlock); + case GetSymbolAddressId: + return handle<GetSymbolAddress>(Channel, *this, + &ThisT::handleGetSymbolAddress); + case GetRemoteInfoId: + return handle<GetRemoteInfo>(Channel, *this, &ThisT::handleGetRemoteInfo); + case ReadMemId: + return handle<ReadMem>(Channel, *this, &ThisT::handleReadMem); + case ReserveMemId: + return handle<ReserveMem>(Channel, *this, &ThisT::handleReserveMem); + case SetProtectionsId: + return handle<SetProtections>(Channel, *this, + &ThisT::handleSetProtections); + case WriteMemId: + return handle<WriteMem>(Channel, *this, &ThisT::handleWriteMem); + case WritePtrId: + return handle<WritePtr>(Channel, *this, &ThisT::handleWritePtr); + default: + return orcError(OrcErrorCode::UnexpectedRPCCall); + } + + llvm_unreachable("Unhandled JIT RPC procedure Id."); + } + + std::error_code requestCompile(TargetAddress &CompiledFnAddr, + TargetAddress TrampolineAddr) { + if (auto EC = call<RequestCompile>(Channel, TrampolineAddr)) + return EC; + + while (1) { + JITProcId Id = InvalidId; + if (auto EC = getNextProcId(Id)) + return EC; + + switch (Id) { + case RequestCompileResponseId: + return handle<RequestCompileResponse>(Channel, + readArgs(CompiledFnAddr)); + default: + if (auto EC = handleKnownProcedure(Id)) + return EC; + } + } + + llvm_unreachable("Fell through request-compile command loop."); + } + +private: + struct Allocator { + Allocator() = default; + Allocator(Allocator &&Other) : Allocs(std::move(Other.Allocs)) {} + Allocator &operator=(Allocator &&Other) { + Allocs = std::move(Other.Allocs); + return *this; + } + + ~Allocator() { + for (auto &Alloc : Allocs) + sys::Memory::releaseMappedMemory(Alloc.second); + } + + std::error_code allocate(void *&Addr, size_t Size, uint32_t Align) { + std::error_code EC; + sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( + Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); + if (EC) + return EC; + + Addr = MB.base(); + assert(Allocs.find(MB.base()) == Allocs.end() && "Duplicate alloc"); + Allocs[MB.base()] = std::move(MB); + return std::error_code(); + } + + std::error_code setProtections(void *block, unsigned Flags) { + auto I = Allocs.find(block); + if (I == Allocs.end()) + return orcError(OrcErrorCode::RemoteMProtectAddrUnrecognized); + return sys::Memory::protectMappedMemory(I->second, Flags); + } + + private: + std::map<void *, sys::MemoryBlock> Allocs; + }; + + static std::error_code doNothing() { return std::error_code(); } + + static TargetAddress reenter(void *JITTargetAddr, void *TrampolineAddr) { + TargetAddress CompiledFnAddr = 0; + + auto T = static_cast<OrcRemoteTargetServer *>(JITTargetAddr); + auto EC = T->requestCompile( + CompiledFnAddr, static_cast<TargetAddress>( + reinterpret_cast<uintptr_t>(TrampolineAddr))); + assert(!EC && "Compile request failed"); + (void)EC; + return CompiledFnAddr; + } + + std::error_code handleCallIntVoid(TargetAddress Addr) { + typedef int (*IntVoidFnTy)(); + IntVoidFnTy Fn = + reinterpret_cast<IntVoidFnTy>(static_cast<uintptr_t>(Addr)); + + DEBUG(dbgs() << " Calling " + << reinterpret_cast<void *>(reinterpret_cast<intptr_t>(Fn)) + << "\n"); + int Result = Fn(); + DEBUG(dbgs() << " Result = " << Result << "\n"); + + return call<CallIntVoidResponse>(Channel, Result); + } + + std::error_code handleCallMain(TargetAddress Addr, + std::vector<std::string> Args) { + typedef int (*MainFnTy)(int, const char *[]); + + MainFnTy Fn = reinterpret_cast<MainFnTy>(static_cast<uintptr_t>(Addr)); + int ArgC = Args.size() + 1; + int Idx = 1; + std::unique_ptr<const char *[]> ArgV(new const char *[ArgC + 1]); + ArgV[0] = "<jit process>"; + for (auto &Arg : Args) + ArgV[Idx++] = Arg.c_str(); + + DEBUG(dbgs() << " Calling " << reinterpret_cast<void *>(Fn) << "\n"); + int Result = Fn(ArgC, ArgV.get()); + DEBUG(dbgs() << " Result = " << Result << "\n"); + + return call<CallMainResponse>(Channel, Result); + } + + std::error_code handleCallVoidVoid(TargetAddress Addr) { + typedef void (*VoidVoidFnTy)(); + VoidVoidFnTy Fn = + reinterpret_cast<VoidVoidFnTy>(static_cast<uintptr_t>(Addr)); + + DEBUG(dbgs() << " Calling " << reinterpret_cast<void *>(Fn) << "\n"); + Fn(); + DEBUG(dbgs() << " Complete.\n"); + + return call<CallVoidVoidResponse>(Channel); + } + + std::error_code handleCreateRemoteAllocator(ResourceIdMgr::ResourceId Id) { + auto I = Allocators.find(Id); + if (I != Allocators.end()) + return orcError(OrcErrorCode::RemoteAllocatorIdAlreadyInUse); + DEBUG(dbgs() << " Created allocator " << Id << "\n"); + Allocators[Id] = Allocator(); + return std::error_code(); + } + + std::error_code handleCreateIndirectStubsOwner(ResourceIdMgr::ResourceId Id) { + auto I = IndirectStubsOwners.find(Id); + if (I != IndirectStubsOwners.end()) + return orcError(OrcErrorCode::RemoteIndirectStubsOwnerIdAlreadyInUse); + DEBUG(dbgs() << " Create indirect stubs owner " << Id << "\n"); + IndirectStubsOwners[Id] = ISBlockOwnerList(); + return std::error_code(); + } + + std::error_code handleDestroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { + auto I = Allocators.find(Id); + if (I == Allocators.end()) + return orcError(OrcErrorCode::RemoteAllocatorDoesNotExist); + Allocators.erase(I); + DEBUG(dbgs() << " Destroyed allocator " << Id << "\n"); + return std::error_code(); + } + + std::error_code + handleDestroyIndirectStubsOwner(ResourceIdMgr::ResourceId Id) { + auto I = IndirectStubsOwners.find(Id); + if (I == IndirectStubsOwners.end()) + return orcError(OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist); + IndirectStubsOwners.erase(I); + return std::error_code(); + } + + std::error_code handleEmitIndirectStubs(ResourceIdMgr::ResourceId Id, + uint32_t NumStubsRequired) { + DEBUG(dbgs() << " ISMgr " << Id << " request " << NumStubsRequired + << " stubs.\n"); + + auto StubOwnerItr = IndirectStubsOwners.find(Id); + if (StubOwnerItr == IndirectStubsOwners.end()) + return orcError(OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist); + + typename TargetT::IndirectStubsInfo IS; + if (auto EC = + TargetT::emitIndirectStubsBlock(IS, NumStubsRequired, nullptr)) + return EC; + + TargetAddress StubsBase = + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(IS.getStub(0))); + TargetAddress PtrsBase = + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(IS.getPtr(0))); + uint32_t NumStubsEmitted = IS.getNumStubs(); + + auto &BlockList = StubOwnerItr->second; + BlockList.push_back(std::move(IS)); + + return call<EmitIndirectStubsResponse>(Channel, StubsBase, PtrsBase, + NumStubsEmitted); + } + + std::error_code handleEmitResolverBlock() { + std::error_code EC; + ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + TargetT::ResolverCodeSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) + return EC; + + TargetT::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()), + &reenter, this); + + return sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), + sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + } + + std::error_code handleEmitTrampolineBlock() { + std::error_code EC; + auto TrampolineBlock = + sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + sys::Process::getPageSize(), nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) + return EC; + + unsigned NumTrampolines = + (sys::Process::getPageSize() - TargetT::PointerSize) / + TargetT::TrampolineSize; + + uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base()); + TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), + NumTrampolines); + + EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), + sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + + TrampolineBlocks.push_back(std::move(TrampolineBlock)); + + return call<EmitTrampolineBlockResponse>( + Channel, + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(TrampolineMem)), + NumTrampolines); + } + + std::error_code handleGetSymbolAddress(const std::string &Name) { + TargetAddress Addr = SymbolLookup(Name); + DEBUG(dbgs() << " Symbol '" << Name << "' = " << format("0x%016x", Addr) + << "\n"); + return call<GetSymbolAddressResponse>(Channel, Addr); + } + + std::error_code handleGetRemoteInfo() { + std::string ProcessTriple = sys::getProcessTriple(); + uint32_t PointerSize = TargetT::PointerSize; + uint32_t PageSize = sys::Process::getPageSize(); + uint32_t TrampolineSize = TargetT::TrampolineSize; + uint32_t IndirectStubSize = TargetT::IndirectStubsInfo::StubSize; + DEBUG(dbgs() << " Remote info:\n" + << " triple = '" << ProcessTriple << "'\n" + << " pointer size = " << PointerSize << "\n" + << " page size = " << PageSize << "\n" + << " trampoline size = " << TrampolineSize << "\n" + << " indirect stub size = " << IndirectStubSize << "\n"); + return call<GetRemoteInfoResponse>(Channel, ProcessTriple, PointerSize, + PageSize, TrampolineSize, + IndirectStubSize); + } + + std::error_code handleReadMem(TargetAddress RSrc, uint64_t Size) { + char *Src = reinterpret_cast<char *>(static_cast<uintptr_t>(RSrc)); + + DEBUG(dbgs() << " Reading " << Size << " bytes from " + << static_cast<void *>(Src) << "\n"); + + if (auto EC = call<ReadMemResponse>(Channel)) + return EC; + + if (auto EC = Channel.appendBytes(Src, Size)) + return EC; + + return Channel.send(); + } + + std::error_code handleReserveMem(ResourceIdMgr::ResourceId Id, uint64_t Size, + uint32_t Align) { + auto I = Allocators.find(Id); + if (I == Allocators.end()) + return orcError(OrcErrorCode::RemoteAllocatorDoesNotExist); + auto &Allocator = I->second; + void *LocalAllocAddr = nullptr; + if (auto EC = Allocator.allocate(LocalAllocAddr, Size, Align)) + return EC; + + DEBUG(dbgs() << " Allocator " << Id << " reserved " << LocalAllocAddr + << " (" << Size << " bytes, alignment " << Align << ")\n"); + + TargetAddress AllocAddr = + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(LocalAllocAddr)); + + return call<ReserveMemResponse>(Channel, AllocAddr); + } + + std::error_code handleSetProtections(ResourceIdMgr::ResourceId Id, + TargetAddress Addr, uint32_t Flags) { + auto I = Allocators.find(Id); + if (I == Allocators.end()) + return orcError(OrcErrorCode::RemoteAllocatorDoesNotExist); + auto &Allocator = I->second; + void *LocalAddr = reinterpret_cast<void *>(static_cast<uintptr_t>(Addr)); + DEBUG(dbgs() << " Allocator " << Id << " set permissions on " << LocalAddr + << " to " << (Flags & sys::Memory::MF_READ ? 'R' : '-') + << (Flags & sys::Memory::MF_WRITE ? 'W' : '-') + << (Flags & sys::Memory::MF_EXEC ? 'X' : '-') << "\n"); + return Allocator.setProtections(LocalAddr, Flags); + } + + std::error_code handleWriteMem(TargetAddress RDst, uint64_t Size) { + char *Dst = reinterpret_cast<char *>(static_cast<uintptr_t>(RDst)); + DEBUG(dbgs() << " Writing " << Size << " bytes to " + << format("0x%016x", RDst) << "\n"); + return Channel.readBytes(Dst, Size); + } + + std::error_code handleWritePtr(TargetAddress Addr, TargetAddress PtrVal) { + DEBUG(dbgs() << " Writing pointer *" << format("0x%016x", Addr) << " = " + << format("0x%016x", PtrVal) << "\n"); + uintptr_t *Ptr = + reinterpret_cast<uintptr_t *>(static_cast<uintptr_t>(Addr)); + *Ptr = static_cast<uintptr_t>(PtrVal); + return std::error_code(); + } + + ChannelT &Channel; + SymbolLookupFtor SymbolLookup; + std::map<ResourceIdMgr::ResourceId, Allocator> Allocators; + typedef std::vector<typename TargetT::IndirectStubsInfo> ISBlockOwnerList; + std::map<ResourceIdMgr::ResourceId, ISBlockOwnerList> IndirectStubsOwners; + sys::OwningMemoryBlock ResolverBlock; + std::vector<sys::OwningMemoryBlock> TrampolineBlocks; +}; + +} // end namespace remote +} // end namespace orc +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif diff --git a/include/llvm/ExecutionEngine/Orc/RPCChannel.h b/include/llvm/ExecutionEngine/Orc/RPCChannel.h new file mode 100644 index 000000000000..b97b6daf5864 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/RPCChannel.h @@ -0,0 +1,179 @@ +// -*- c++ -*- + +#ifndef LLVM_EXECUTIONENGINE_ORC_RPCCHANNEL_H +#define LLVM_EXECUTIONENGINE_ORC_RPCCHANNEL_H + +#include "OrcError.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Endian.h" + +#include <system_error> + +namespace llvm { +namespace orc { +namespace remote { + +/// Interface for byte-streams to be used with RPC. +class RPCChannel { +public: + virtual ~RPCChannel() {} + + /// Read Size bytes from the stream into *Dst. + virtual std::error_code readBytes(char *Dst, unsigned Size) = 0; + + /// Read size bytes from *Src and append them to the stream. + virtual std::error_code appendBytes(const char *Src, unsigned Size) = 0; + + /// Flush the stream if possible. + virtual std::error_code send() = 0; +}; + +/// RPC channel serialization for a variadic list of arguments. +template <typename T, typename... Ts> +std::error_code serialize_seq(RPCChannel &C, const T &Arg, const Ts &... Args) { + if (auto EC = serialize(C, Arg)) + return EC; + return serialize_seq(C, Args...); +} + +/// RPC channel serialization for an (empty) variadic list of arguments. +inline std::error_code serialize_seq(RPCChannel &C) { + return std::error_code(); +} + +/// RPC channel deserialization for a variadic list of arguments. +template <typename T, typename... Ts> +std::error_code deserialize_seq(RPCChannel &C, T &Arg, Ts &... Args) { + if (auto EC = deserialize(C, Arg)) + return EC; + return deserialize_seq(C, Args...); +} + +/// RPC channel serialization for an (empty) variadic list of arguments. +inline std::error_code deserialize_seq(RPCChannel &C) { + return std::error_code(); +} + +/// RPC channel serialization for integer primitives. +template <typename T> +typename std::enable_if< + std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value || + std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value || + std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value || + std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value, + std::error_code>::type +serialize(RPCChannel &C, T V) { + support::endian::byte_swap<T, support::big>(V); + return C.appendBytes(reinterpret_cast<const char *>(&V), sizeof(T)); +} + +/// RPC channel deserialization for integer primitives. +template <typename T> +typename std::enable_if< + std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value || + std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value || + std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value || + std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value, + std::error_code>::type +deserialize(RPCChannel &C, T &V) { + if (auto EC = C.readBytes(reinterpret_cast<char *>(&V), sizeof(T))) + return EC; + support::endian::byte_swap<T, support::big>(V); + return std::error_code(); +} + +/// RPC channel serialization for enums. +template <typename T> +typename std::enable_if<std::is_enum<T>::value, std::error_code>::type +serialize(RPCChannel &C, T V) { + return serialize(C, static_cast<typename std::underlying_type<T>::type>(V)); +} + +/// RPC channel deserialization for enums. +template <typename T> +typename std::enable_if<std::is_enum<T>::value, std::error_code>::type +deserialize(RPCChannel &C, T &V) { + typename std::underlying_type<T>::type Tmp; + std::error_code EC = deserialize(C, Tmp); + V = static_cast<T>(Tmp); + return EC; +} + +/// RPC channel serialization for bools. +inline std::error_code serialize(RPCChannel &C, bool V) { + uint8_t VN = V ? 1 : 0; + return C.appendBytes(reinterpret_cast<const char *>(&VN), 1); +} + +/// RPC channel deserialization for bools. +inline std::error_code deserialize(RPCChannel &C, bool &V) { + uint8_t VN = 0; + if (auto EC = C.readBytes(reinterpret_cast<char *>(&VN), 1)) + return EC; + + V = (VN != 0) ? true : false; + return std::error_code(); +} + +/// RPC channel serialization for StringRefs. +/// Note: There is no corresponding deseralization for this, as StringRef +/// doesn't own its memory and so can't hold the deserialized data. +inline std::error_code serialize(RPCChannel &C, StringRef S) { + if (auto EC = serialize(C, static_cast<uint64_t>(S.size()))) + return EC; + return C.appendBytes((const char *)S.bytes_begin(), S.size()); +} + +/// RPC channel serialization for std::strings. +inline std::error_code serialize(RPCChannel &C, const std::string &S) { + return serialize(C, StringRef(S)); +} + +/// RPC channel deserialization for std::strings. +inline std::error_code deserialize(RPCChannel &C, std::string &S) { + uint64_t Count; + if (auto EC = deserialize(C, Count)) + return EC; + S.resize(Count); + return C.readBytes(&S[0], Count); +} + +/// RPC channel serialization for ArrayRef<T>. +template <typename T> +std::error_code serialize(RPCChannel &C, const ArrayRef<T> &A) { + if (auto EC = serialize(C, static_cast<uint64_t>(A.size()))) + return EC; + + for (const auto &E : A) + if (auto EC = serialize(C, E)) + return EC; + + return std::error_code(); +} + +/// RPC channel serialization for std::array<T>. +template <typename T> +std::error_code serialize(RPCChannel &C, const std::vector<T> &V) { + return serialize(C, ArrayRef<T>(V)); +} + +/// RPC channel deserialization for std::array<T>. +template <typename T> +std::error_code deserialize(RPCChannel &C, std::vector<T> &V) { + uint64_t Count = 0; + if (auto EC = deserialize(C, Count)) + return EC; + + V.resize(Count); + for (auto &E : V) + if (auto EC = deserialize(C, E)) + return EC; + + return std::error_code(); +} + +} // end namespace remote +} // end namespace orc +} // end namespace llvm + +#endif diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h new file mode 100644 index 000000000000..0bd5cbc0cdde --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -0,0 +1,266 @@ +//===----- RPCUTils.h - Basic tilities for building RPC APIs ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Basic utilities for building RPC APIs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_RPCUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_RPCUTILS_H + +#include "llvm/ADT/STLExtras.h" + +namespace llvm { +namespace orc { +namespace remote { + +// Base class containing utilities that require partial specialization. +// These cannot be included in RPC, as template class members cannot be +// partially specialized. +class RPCBase { +protected: + template <typename ProcedureIdT, ProcedureIdT ProcId, typename... Ts> + class ProcedureHelper { + public: + static const ProcedureIdT Id = ProcId; + }; + + template <typename ChannelT, typename Proc> class CallHelper; + + template <typename ChannelT, typename ProcedureIdT, ProcedureIdT ProcId, + typename... ArgTs> + class CallHelper<ChannelT, ProcedureHelper<ProcedureIdT, ProcId, ArgTs...>> { + public: + static std::error_code call(ChannelT &C, const ArgTs &... Args) { + if (auto EC = serialize(C, ProcId)) + return EC; + // If you see a compile-error on this line you're probably calling a + // function with the wrong signature. + return serialize_seq(C, Args...); + } + }; + + template <typename ChannelT, typename Proc> class HandlerHelper; + + template <typename ChannelT, typename ProcedureIdT, ProcedureIdT ProcId, + typename... ArgTs> + class HandlerHelper<ChannelT, + ProcedureHelper<ProcedureIdT, ProcId, ArgTs...>> { + public: + template <typename HandlerT> + static std::error_code handle(ChannelT &C, HandlerT Handler) { + return readAndHandle(C, Handler, llvm::index_sequence_for<ArgTs...>()); + } + + private: + template <typename HandlerT, size_t... Is> + static std::error_code readAndHandle(ChannelT &C, HandlerT Handler, + llvm::index_sequence<Is...> _) { + std::tuple<ArgTs...> RPCArgs; + if (auto EC = deserialize_seq(C, std::get<Is>(RPCArgs)...)) + return EC; + return Handler(std::get<Is>(RPCArgs)...); + } + }; + + template <typename ClassT, typename... ArgTs> class MemberFnWrapper { + public: + typedef std::error_code (ClassT::*MethodT)(ArgTs...); + MemberFnWrapper(ClassT &Instance, MethodT Method) + : Instance(Instance), Method(Method) {} + std::error_code operator()(ArgTs &... Args) { + return (Instance.*Method)(Args...); + } + + private: + ClassT &Instance; + MethodT Method; + }; + + template <typename... ArgTs> class ReadArgs { + public: + std::error_code operator()() { return std::error_code(); } + }; + + template <typename ArgT, typename... ArgTs> + class ReadArgs<ArgT, ArgTs...> : public ReadArgs<ArgTs...> { + public: + ReadArgs(ArgT &Arg, ArgTs &... Args) + : ReadArgs<ArgTs...>(Args...), Arg(Arg) {} + + std::error_code operator()(ArgT &ArgVal, ArgTs &... ArgVals) { + this->Arg = std::move(ArgVal); + return ReadArgs<ArgTs...>::operator()(ArgVals...); + } + + private: + ArgT &Arg; + }; +}; + +/// Contains primitive utilities for defining, calling and handling calls to +/// remote procedures. ChannelT is a bidirectional stream conforming to the +/// RPCChannel interface (see RPCChannel.h), and ProcedureIdT is a procedure +/// identifier type that must be serializable on ChannelT. +/// +/// These utilities support the construction of very primitive RPC utilities. +/// Their intent is to ensure correct serialization and deserialization of +/// procedure arguments, and to keep the client and server's view of the API in +/// sync. +/// +/// These utilities do not support return values. These can be handled by +/// declaring a corresponding '.*Response' procedure and expecting it after a +/// call). They also do not support versioning: the client and server *must* be +/// compiled with the same procedure definitions. +/// +/// +/// +/// Overview (see comments individual types/methods for details): +/// +/// Procedure<Id, Args...> : +/// +/// associates a unique serializable id with an argument list. +/// +/// +/// call<Proc>(Channel, Args...) : +/// +/// Calls the remote procedure 'Proc' by serializing Proc's id followed by its +/// arguments and sending the resulting bytes to 'Channel'. +/// +/// +/// handle<Proc>(Channel, <functor matching std::error_code(Args...)> : +/// +/// Handles a call to 'Proc' by deserializing its arguments and calling the +/// given functor. This assumes that the id for 'Proc' has already been +/// deserialized. +/// +/// expect<Proc>(Channel, <functor matching std::error_code(Args...)> : +/// +/// The same as 'handle', except that the procedure id should not have been +/// read yet. Expect will deserialize the id and assert that it matches Proc's +/// id. If it does not, and unexpected RPC call error is returned. + +template <typename ChannelT, typename ProcedureIdT = uint32_t> +class RPC : public RPCBase { +public: + /// Utility class for defining/referring to RPC procedures. + /// + /// Typedefs of this utility are used when calling/handling remote procedures. + /// + /// ProcId should be a unique value of ProcedureIdT (i.e. not used with any + /// other Procedure typedef in the RPC API being defined. + /// + /// the template argument Ts... gives the argument list for the remote + /// procedure. + /// + /// E.g. + /// + /// typedef Procedure<0, bool> Proc1; + /// typedef Procedure<1, std::string, std::vector<int>> Proc2; + /// + /// if (auto EC = call<Proc1>(Channel, true)) + /// /* handle EC */; + /// + /// if (auto EC = expect<Proc2>(Channel, + /// [](std::string &S, std::vector<int> &V) { + /// // Stuff. + /// return std::error_code(); + /// }) + /// /* handle EC */; + /// + template <ProcedureIdT ProcId, typename... Ts> + using Procedure = ProcedureHelper<ProcedureIdT, ProcId, Ts...>; + + /// Serialize Args... to channel C, but do not call C.send(). + /// + /// For buffered channels, this can be used to queue up several calls before + /// flushing the channel. + template <typename Proc, typename... ArgTs> + static std::error_code appendCall(ChannelT &C, const ArgTs &... Args) { + return CallHelper<ChannelT, Proc>::call(C, Args...); + } + + /// Serialize Args... to channel C and call C.send(). + template <typename Proc, typename... ArgTs> + static std::error_code call(ChannelT &C, const ArgTs &... Args) { + if (auto EC = appendCall<Proc>(C, Args...)) + return EC; + return C.send(); + } + + /// Deserialize and return an enum whose underlying type is ProcedureIdT. + static std::error_code getNextProcId(ChannelT &C, ProcedureIdT &Id) { + return deserialize(C, Id); + } + + /// Deserialize args for Proc from C and call Handler. The signature of + /// handler must conform to 'std::error_code(Args...)' where Args... matches + /// the arguments used in the Proc typedef. + template <typename Proc, typename HandlerT> + static std::error_code handle(ChannelT &C, HandlerT Handler) { + return HandlerHelper<ChannelT, Proc>::handle(C, Handler); + } + + /// Helper version of 'handle' for calling member functions. + template <typename Proc, typename ClassT, typename... ArgTs> + static std::error_code + handle(ChannelT &C, ClassT &Instance, + std::error_code (ClassT::*HandlerMethod)(ArgTs...)) { + return handle<Proc>( + C, MemberFnWrapper<ClassT, ArgTs...>(Instance, HandlerMethod)); + } + + /// Deserialize a ProcedureIdT from C and verify it matches the id for Proc. + /// If the id does match, deserialize the arguments and call the handler + /// (similarly to handle). + /// If the id does not match, return an unexpect RPC call error and do not + /// deserialize any further bytes. + template <typename Proc, typename HandlerT> + static std::error_code expect(ChannelT &C, HandlerT Handler) { + ProcedureIdT ProcId; + if (auto EC = getNextProcId(C, ProcId)) + return EC; + if (ProcId != Proc::Id) + return orcError(OrcErrorCode::UnexpectedRPCCall); + return handle<Proc>(C, Handler); + } + + /// Helper version of expect for calling member functions. + template <typename Proc, typename ClassT, typename... ArgTs> + static std::error_code + expect(ChannelT &C, ClassT &Instance, + std::error_code (ClassT::*HandlerMethod)(ArgTs...)) { + return expect<Proc>( + C, MemberFnWrapper<ClassT, ArgTs...>(Instance, HandlerMethod)); + } + + /// Helper for handling setter procedures - this method returns a functor that + /// sets the variables referred to by Args... to values deserialized from the + /// channel. + /// E.g. + /// + /// typedef Procedure<0, bool, int> Proc1; + /// + /// ... + /// bool B; + /// int I; + /// if (auto EC = expect<Proc1>(Channel, readArgs(B, I))) + /// /* Handle Args */ ; + /// + template <typename... ArgTs> + static ReadArgs<ArgTs...> readArgs(ArgTs &... Args) { + return ReadArgs<ArgTs...>(Args...); + } +}; + +} // end namespace remote +} // end namespace orc +} // end namespace llvm + +#endif diff --git a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h index 207bad06c239..c5006962550e 100644 --- a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h +++ b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h @@ -30,6 +30,10 @@ class ExecutionEngine; class MCJITMemoryManager : public RuntimeDyld::MemoryManager { public: + + // Don't hide the notifyObjectLoaded method from RuntimeDyld::MemoryManager. + using RuntimeDyld::MemoryManager::notifyObjectLoaded; + /// This method is called after an object has been loaded into memory but /// before relocations are applied to the loaded sections. The object load /// may have been initiated by MCJIT to resolve an external symbol for another diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h index 385b8d0a30b1..100e97b8b3d9 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -95,7 +95,9 @@ public: /// \brief Memory Management. class MemoryManager { + friend class RuntimeDyld; public: + MemoryManager() : FinalizationLocked(false) {} virtual ~MemoryManager() {} /// Allocate a memory block of (at least) the given size suitable for @@ -122,9 +124,11 @@ public: /// /// Note that by default the callback is disabled. To enable it /// redefine the method needsToReserveAllocationSpace to return true. - virtual void reserveAllocationSpace(uintptr_t CodeSize, - uintptr_t DataSizeRO, - uintptr_t DataSizeRW) {} + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) {} /// Override to return true to enable the reserveAllocationSpace callback. virtual bool needsToReserveAllocationSpace() { return false; } @@ -151,8 +155,23 @@ public: /// Returns true if an error occurred, false otherwise. virtual bool finalizeMemory(std::string *ErrMsg = nullptr) = 0; + /// This method is called after an object has been loaded into memory but + /// before relocations are applied to the loaded sections. + /// + /// Memory managers which are preparing code for execution in an external + /// address space can use this call to remap the section addresses for the + /// newly loaded object. + /// + /// For clients that do not need access to an ExecutionEngine instance this + /// method should be preferred to its cousin + /// MCJITMemoryManager::notifyObjectLoaded as this method is compatible with + /// ORC JIT stacks. + virtual void notifyObjectLoaded(RuntimeDyld &RTDyld, + const object::ObjectFile &Obj) {} + private: virtual void anchor(); + bool FinalizationLocked; }; /// \brief Symbol resolution. @@ -241,6 +260,25 @@ public: this->ProcessAllSections = ProcessAllSections; } + /// Perform all actions needed to make the code owned by this RuntimeDyld + /// instance executable: + /// + /// 1) Apply relocations. + /// 2) Register EH frames. + /// 3) Update memory permissions*. + /// + /// * Finalization is potentially recursive**, and the 3rd step will only be + /// applied by the outermost call to finalize. This allows different + /// RuntimeDyld instances to share a memory manager without the innermost + /// finalization locking the memory and causing relocation fixup errors in + /// outer instances. + /// + /// ** Recursive finalization occurs when one RuntimeDyld instances needs the + /// address of a symbol owned by some other instance in order to apply + /// relocations. + /// + void finalizeWithMemoryManagerLocking(); + private: // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public // interface. diff --git a/include/llvm/IR/Attributes.td b/include/llvm/IR/Attributes.td index 797cd55427b3..30249bbd8fab 100644 --- a/include/llvm/IR/Attributes.td +++ b/include/llvm/IR/Attributes.td @@ -189,4 +189,9 @@ class MergeRule<string F> { string MergeFunc = F; } +def : MergeRule<"setAND<LessPreciseFPMADAttr>">; +def : MergeRule<"setAND<NoInfsFPMathAttr>">; +def : MergeRule<"setAND<NoNansFPMathAttr>">; +def : MergeRule<"setAND<UnsafeFPMathAttr>">; +def : MergeRule<"setOR<NoImplicitFloatAttr>">; def : MergeRule<"adjustCallerSSPLevel">; diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h index 2a983930bf4d..4f64caeade20 100644 --- a/include/llvm/IR/Function.h +++ b/include/llvm/IR/Function.h @@ -66,7 +66,8 @@ private: * bit 2 : HasPrologueData * bit 3 : HasPersonalityFn * bits 4-13 : CallingConvention - * bits 14-15 : [reserved] + * bits 14 : HasGC + * bits 15 : [reserved] */ /// Bits from GlobalObject::GlobalObjectSubclassData. @@ -220,9 +221,11 @@ public: /// hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm /// to use during code generation. - bool hasGC() const; - const char *getGC() const; - void setGC(const char *Str); + bool hasGC() const { + return getSubclassDataFromValue() & (1<<14); + } + const std::string &getGC() const; + void setGC(const std::string Str); void clearGC(); /// @brief adds the attribute to the list of attributes. diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index a30505471aac..1b75c60631b0 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -178,10 +178,10 @@ public: void clearFastMathFlags() { FMF.clear(); } /// \brief Set the floating point math metadata to be used. - void SetDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; } + void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; } /// \brief Set the fast-math flags to be used with generated fp-math operators - void SetFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; } + void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; } //===--------------------------------------------------------------------===// // RAII helpers. diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index 5a95ddced538..f67029ab56e3 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -575,7 +575,7 @@ def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty], def int_experimental_gc_result : Intrinsic<[llvm_any_ty], [llvm_token_ty], [IntrReadMem]>; -def int_experimental_gc_relocate : Intrinsic<[llvm_anyptr_ty], +def int_experimental_gc_relocate : Intrinsic<[llvm_any_ty], [llvm_token_ty, llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>; diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td index 54bcbd8da509..8023a9f6e8e9 100644 --- a/include/llvm/IR/IntrinsicsX86.td +++ b/include/llvm/IR/IntrinsicsX86.td @@ -1507,6 +1507,60 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pshuf_d_128 : + GCCBuiltin<"__builtin_ia32_pshufd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_i16_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshuf_d_256 : + GCCBuiltin<"__builtin_ia32_pshufd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_i16_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshuf_d_512 : + GCCBuiltin<"__builtin_ia32_pshufd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufh_w_128 : + GCCBuiltin<"__builtin_ia32_pshufhw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufh_w_256 : + GCCBuiltin<"__builtin_ia32_pshufhw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufh_w_512 : + GCCBuiltin<"__builtin_ia32_pshufhw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufl_w_128 : + GCCBuiltin<"__builtin_ia32_pshuflw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufl_w_256 : + GCCBuiltin<"__builtin_ia32_pshuflw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_pshufl_w_512 : + GCCBuiltin<"__builtin_ia32_pshuflw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_shuf_f32x4_256 : GCCBuiltin<"__builtin_ia32_shuf_f32x4_256_mask">, Intrinsic<[llvm_v8f32_ty], @@ -1836,25 +1890,69 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_maskload_ps_256 : GCCBuiltin<"__builtin_ia32_maskloadps256">, Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8i32_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_ps_512 : GCCBuiltin<"__builtin_ia32_loadups512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_pd_512 : GCCBuiltin<"__builtin_ia32_loadupd512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_load_ps_512 : GCCBuiltin<"__builtin_ia32_loadaps512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_load_pd_512 : GCCBuiltin<"__builtin_ia32_loadapd512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_move_ss : GCCBuiltin<"__builtin_ia32_movss_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_move_sd : GCCBuiltin<"__builtin_ia32_movsd_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; + def int_x86_avx512_mask_loadu_ps_128 : + GCCBuiltin<"__builtin_ia32_loadups128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_loadu_ps_256 : + GCCBuiltin<"__builtin_ia32_loadups256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_ptr_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_loadu_ps_512 : + GCCBuiltin<"__builtin_ia32_loadups512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_loadu_pd_128 : + GCCBuiltin<"__builtin_ia32_loadupd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_ptr_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_loadu_pd_256 : + GCCBuiltin<"__builtin_ia32_loadupd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_ptr_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_loadu_pd_512 : + GCCBuiltin<"__builtin_ia32_loadupd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_load_ps_128 : + GCCBuiltin<"__builtin_ia32_loadaps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_load_ps_256 : + GCCBuiltin<"__builtin_ia32_loadaps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_ptr_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_load_ps_512 : + GCCBuiltin<"__builtin_ia32_loadaps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_load_pd_128 : + GCCBuiltin<"__builtin_ia32_loadapd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_ptr_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_load_pd_256 : + GCCBuiltin<"__builtin_ia32_loadapd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_ptr_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + def int_x86_avx512_mask_load_pd_512 : + GCCBuiltin<"__builtin_ia32_loadapd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_move_ss : + GCCBuiltin<"__builtin_ia32_movss_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_move_sd : + GCCBuiltin<"__builtin_ia32_movsd_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; } // Conditional store ops @@ -2262,6 +2360,46 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_w_128 : GCCBuiltin<"__builtin_ia32_psllw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_w_256 : GCCBuiltin<"__builtin_ia32_psllw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v8i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_w_512 : GCCBuiltin<"__builtin_ia32_psllw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_wi_128 : GCCBuiltin<"__builtin_ia32_psllwi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_wi_256 : GCCBuiltin<"__builtin_ia32_psllwi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_wi_512 : GCCBuiltin<"__builtin_ia32_psllwi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv16_hi : GCCBuiltin<"__builtin_ia32_psllv16hi_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv2_di : GCCBuiltin<"__builtin_ia32_psllv2di_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv32hi : GCCBuiltin<"__builtin_ia32_psllv32hi_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv4_di : GCCBuiltin<"__builtin_ia32_psllv4di_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv4_si : GCCBuiltin<"__builtin_ia32_psllv4si_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv8_hi : GCCBuiltin<"__builtin_ia32_psllv8hi_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psllv8_si : GCCBuiltin<"__builtin_ia32_psllv8si_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psra_d_128 : GCCBuiltin<"__builtin_ia32_psrad128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; @@ -2823,6 +2961,28 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav16_hi : GCCBuiltin<"__builtin_ia32_psrav16hi_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav32_hi : GCCBuiltin<"__builtin_ia32_psrav32hi_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav4_si : GCCBuiltin<"__builtin_ia32_psrav4si_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav8_hi : GCCBuiltin<"__builtin_ia32_psrav8hi_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav8_si : GCCBuiltin<"__builtin_ia32_psrav8si_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav_q_128 : GCCBuiltin<"__builtin_ia32_psravq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrav_q_256 : GCCBuiltin<"__builtin_ia32_psravq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrlv16_hi : GCCBuiltin<"__builtin_ia32_psrlv16hi_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; @@ -2844,6 +3004,83 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_psrlv8_si : GCCBuiltin<"__builtin_ia32_psrlv8si_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_prorv_d_128 : GCCBuiltin<"__builtin_ia32_prorvd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prorv_d_256 : GCCBuiltin<"__builtin_ia32_prorvd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prorv_d_512 : GCCBuiltin<"__builtin_ia32_prorvd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prorv_q_128 : GCCBuiltin<"__builtin_ia32_prorvq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prorv_q_256 : GCCBuiltin<"__builtin_ia32_prorvq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prorv_q_512 : GCCBuiltin<"__builtin_ia32_prorvq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_prol_d_128 : GCCBuiltin<"__builtin_ia32_prold128_mask">, + Intrinsic<[llvm_v4i32_ty] , [llvm_v4i32_ty, + llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prol_d_256 : GCCBuiltin<"__builtin_ia32_prold256_mask">, + Intrinsic<[llvm_v8i32_ty] , [llvm_v8i32_ty, + llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prol_d_512 : GCCBuiltin<"__builtin_ia32_prold512_mask">, + Intrinsic<[llvm_v16i32_ty] , [llvm_v16i32_ty, + llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prol_q_128 : GCCBuiltin<"__builtin_ia32_prolq128_mask">, + Intrinsic<[llvm_v2i64_ty] , [llvm_v2i64_ty, + llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prol_q_256 : GCCBuiltin<"__builtin_ia32_prolq256_mask">, + Intrinsic<[llvm_v4i64_ty] , [llvm_v4i64_ty, + llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prol_q_512 : GCCBuiltin<"__builtin_ia32_prolq512_mask">, + Intrinsic<[llvm_v8i64_ty] , [llvm_v8i64_ty, + llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + + def int_x86_avx512_mask_prolv_d_128 : GCCBuiltin<"__builtin_ia32_prolvd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prolv_d_256 : GCCBuiltin<"__builtin_ia32_prolvd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prolv_d_512 : GCCBuiltin<"__builtin_ia32_prolvd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prolv_q_128 : GCCBuiltin<"__builtin_ia32_prolvq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prolv_q_256 : GCCBuiltin<"__builtin_ia32_prolvq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_prolv_q_512 : GCCBuiltin<"__builtin_ia32_prolvq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_d_128 : GCCBuiltin<"__builtin_ia32_prord128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_d_256 : GCCBuiltin<"__builtin_ia32_prord256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_d_512 : GCCBuiltin<"__builtin_ia32_prord512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_q_128 : GCCBuiltin<"__builtin_ia32_prorq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_q_256 : GCCBuiltin<"__builtin_ia32_prorq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pror_q_512 : GCCBuiltin<"__builtin_ia32_prorq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + } // Gather ops @@ -4208,6 +4445,61 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_kortestc_w : GCCBuiltin<"__builtin_ia32_kortestchi">, Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pmovsxb_d_128 : GCCBuiltin<"__builtin_ia32_pmovsxbd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_d_256 : GCCBuiltin<"__builtin_ia32_pmovsxbd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v16i8_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_d_512 : GCCBuiltin<"__builtin_ia32_pmovsxbd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i8_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_q_128 : GCCBuiltin<"__builtin_ia32_pmovsxbq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_q_256 : GCCBuiltin<"__builtin_ia32_pmovsxbq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v16i8_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_q_512 : GCCBuiltin<"__builtin_ia32_pmovsxbq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v16i8_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_w_128 : GCCBuiltin<"__builtin_ia32_pmovsxbw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_w_256 : GCCBuiltin<"__builtin_ia32_pmovsxbw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i8_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxb_w_512 : GCCBuiltin<"__builtin_ia32_pmovsxbw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i8_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxd_q_128 : GCCBuiltin<"__builtin_ia32_pmovsxdq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxd_q_256 : GCCBuiltin<"__builtin_ia32_pmovsxdq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i32_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxd_q_512 : GCCBuiltin<"__builtin_ia32_pmovsxdq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i32_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_d_128 : GCCBuiltin<"__builtin_ia32_pmovsxwd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_d_256 : GCCBuiltin<"__builtin_ia32_pmovsxwd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i16_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_d_512 : GCCBuiltin<"__builtin_ia32_pmovsxwd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i16_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_q_128 : GCCBuiltin<"__builtin_ia32_pmovsxwq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_q_256 : GCCBuiltin<"__builtin_ia32_pmovsxwq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v8i16_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovsxw_q_512 : GCCBuiltin<"__builtin_ia32_pmovsxwq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i16_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } // Conversion ops @@ -5319,6 +5611,62 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_pmovzxdq : GCCBuiltin<"__builtin_ia32_pmovzxdq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pmovzxb_d_128 : GCCBuiltin<"__builtin_ia32_pmovzxbd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_d_256 : GCCBuiltin<"__builtin_ia32_pmovzxbd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v16i8_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_d_512 : GCCBuiltin<"__builtin_ia32_pmovzxbd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i8_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_q_128 : GCCBuiltin<"__builtin_ia32_pmovzxbq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_q_256 : GCCBuiltin<"__builtin_ia32_pmovzxbq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v16i8_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_q_512 : GCCBuiltin<"__builtin_ia32_pmovzxbq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v16i8_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_w_128 : GCCBuiltin<"__builtin_ia32_pmovzxbw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_w_256 : GCCBuiltin<"__builtin_ia32_pmovzxbw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i8_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxb_w_512 : GCCBuiltin<"__builtin_ia32_pmovzxbw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i8_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxd_q_128 : GCCBuiltin<"__builtin_ia32_pmovzxdq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxd_q_256 : GCCBuiltin<"__builtin_ia32_pmovzxdq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i32_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxd_q_512 : GCCBuiltin<"__builtin_ia32_pmovzxdq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i32_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_d_128 : GCCBuiltin<"__builtin_ia32_pmovzxwd128_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_d_256 : GCCBuiltin<"__builtin_ia32_pmovzxwd256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i16_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_d_512 : GCCBuiltin<"__builtin_ia32_pmovzxwd512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i16_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_q_128 : GCCBuiltin<"__builtin_ia32_pmovzxwq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_q_256 : GCCBuiltin<"__builtin_ia32_pmovzxwq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v8i16_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmovzxw_q_512 : GCCBuiltin<"__builtin_ia32_pmovzxwq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i16_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + } //Bitwise Ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h index c546fc3d1ee0..56aa3010d925 100644 --- a/include/llvm/IR/LLVMContext.h +++ b/include/llvm/IR/LLVMContext.h @@ -93,6 +93,17 @@ public: /// tag registered with an LLVMContext has an unique ID. uint32_t getOperandBundleTagID(StringRef Tag) const; + + /// Define the GC for a function + void setGC(const Function &Fn, std::string GCName); + + /// Return the GC for a function + const std::string &getGC(const Function &Fn); + + /// Remove the GC for a function + void deleteGC(const Function &Fn); + + typedef void (*InlineAsmDiagHandlerTy)(const SMDiagnostic&, void *Context, unsigned LocCookie); diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h index 4a8557d074f0..df8ce354bb7f 100644 --- a/include/llvm/IR/Metadata.h +++ b/include/llvm/IR/Metadata.h @@ -915,11 +915,21 @@ public: /// \brief Resolve cycles. /// /// Once all forward declarations have been resolved, force cycles to be - /// resolved. If \p AllowTemps is true, then any temporary metadata - /// is ignored, otherwise it asserts when encountering temporary metadata. + /// resolved. This interface is used when there are no more temporaries, + /// and thus unresolved nodes are part of cycles and no longer need RAUW + /// support. /// /// \pre No operands (or operands' operands, etc.) have \a isTemporary(). - void resolveCycles(bool AllowTemps = false); + void resolveCycles() { resolveRecursivelyImpl(/* AllowTemps */ false); } + + /// \brief Resolve cycles while ignoring temporaries. + /// + /// This drops RAUW support for any temporaries, which can no longer + /// be uniqued. + /// + void resolveNonTemporaries() { + resolveRecursivelyImpl(/* AllowTemps */ true); + } /// \brief Replace a temporary node with a permanent one. /// @@ -977,6 +987,11 @@ private: void decrementUnresolvedOperandCount(); unsigned countUnresolvedOperands(); + /// Resolve cycles recursively. If \p AllowTemps is true, then any temporary + /// metadata is ignored, otherwise it asserts when encountering temporary + /// metadata. + void resolveRecursivelyImpl(bool AllowTemps); + /// \brief Mutate this to be "uniqued". /// /// Mutate this so that \a isUniqued(). diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index cb2b1394e92b..90fbc1d891b7 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -132,7 +132,6 @@ void initializeEarlyCSELegacyPassPass(PassRegistry &); void initializeEliminateAvailableExternallyPass(PassRegistry&); void initializeExpandISelPseudosPass(PassRegistry&); void initializeForceFunctionAttrsLegacyPassPass(PassRegistry&); -void initializeFunctionAttrsPass(PassRegistry&); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGVNPass(PassRegistry&); @@ -227,6 +226,7 @@ void initializePostDomOnlyViewerPass(PassRegistry&); void initializePostDomPrinterPass(PassRegistry&); void initializePostDomViewerPass(PassRegistry&); void initializePostDominatorTreePass(PassRegistry&); +void initializePostOrderFunctionAttrsPass(PassRegistry&); void initializePostRASchedulerPass(PassRegistry&); void initializePostMachineSchedulerPass(PassRegistry&); void initializePrintFunctionPassWrapperPass(PassRegistry&); @@ -242,6 +242,7 @@ void initializeRegionOnlyPrinterPass(PassRegistry&); void initializeRegionOnlyViewerPass(PassRegistry&); void initializeRegionPrinterPass(PassRegistry&); void initializeRegionViewerPass(PassRegistry&); +void initializeReversePostOrderFunctionAttrsPass(PassRegistry&); void initializeRewriteStatepointsForGCPass(PassRegistry&); void initializeSafeStackPass(PassRegistry&); void initializeSCCPPass(PassRegistry&); diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h index 29fcd93a2a1c..d695d11a6369 100644 --- a/include/llvm/LinkAllPasses.h +++ b/include/llvm/LinkAllPasses.h @@ -157,7 +157,8 @@ namespace { (void) llvm::createPostDomTree(); (void) llvm::createInstructionNamerPass(); (void) llvm::createMetaRenamerPass(); - (void) llvm::createFunctionAttrsPass(); + (void) llvm::createPostOrderFunctionAttrsPass(); + (void) llvm::createReversePostOrderFunctionAttrsPass(); (void) llvm::createMergeFunctionsPass(); (void) llvm::createPrintModulePass(*(llvm::raw_ostream*)nullptr); (void) llvm::createPrintFunctionPass(*(llvm::raw_ostream*)nullptr); diff --git a/include/llvm/Linker/Linker.h b/include/llvm/Linker/Linker.h index dde3f73883ca..2b051e6d15c9 100644 --- a/include/llvm/Linker/Linker.h +++ b/include/llvm/Linker/Linker.h @@ -67,10 +67,9 @@ public: DenseMap<unsigned, MDNode *> *ValIDToTempMDMap); }; -/// Create a new module with exported local functions renamed and promoted -/// for ThinLTO. -std::unique_ptr<Module> renameModuleForThinLTO(std::unique_ptr<Module> M, - const FunctionInfoIndex *Index); +/// Perform in-place global value handling on the given Module for +/// exported local functions renamed and promoted for ThinLTO. +bool renameModuleForThinLTO(Module &M, const FunctionInfoIndex *Index); } // End llvm namespace diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h index 1d6bdef0af27..f6ccdc095551 100644 --- a/include/llvm/MC/MCExpr.h +++ b/include/llvm/MC/MCExpr.h @@ -290,6 +290,9 @@ public: VK_Hexagon_LD_PLT, VK_Hexagon_IE, VK_Hexagon_IE_GOT, + + VK_WebAssembly_FUNCTION, // Function table index, rather than virtual addr + VK_TPREL, VK_DTPREL }; diff --git a/include/llvm/MC/MCObjectFileInfo.h b/include/llvm/MC/MCObjectFileInfo.h index cf2c3f12bb6b..8a3a6af3bf79 100644 --- a/include/llvm/MC/MCObjectFileInfo.h +++ b/include/llvm/MC/MCObjectFileInfo.h @@ -92,6 +92,7 @@ protected: MCSection *DwarfLocSection; MCSection *DwarfARangesSection; MCSection *DwarfRangesSection; + MCSection *DwarfMacinfoSection; // The pubnames section is no longer generated by default. The generation // can be enabled by a compiler flag. MCSection *DwarfPubNamesSection; @@ -245,6 +246,7 @@ public: MCSection *getDwarfLocSection() const { return DwarfLocSection; } MCSection *getDwarfARangesSection() const { return DwarfARangesSection; } MCSection *getDwarfRangesSection() const { return DwarfRangesSection; } + MCSection *getDwarfMacinfoSection() const { return DwarfMacinfoSection; } // DWARF5 Experimental Debug Info Sections MCSection *getDwarfAccelNamesSection() const { diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h index 494f02dfad3e..04d143ffef66 100644 --- a/include/llvm/MC/MCStreamer.h +++ b/include/llvm/MC/MCStreamer.h @@ -131,6 +131,10 @@ public: void finish() override; + /// Reset any state between object emissions, i.e. the equivalent of + /// MCStreamer's reset method. + virtual void reset(); + /// Callback used to implement the ldr= pseudo. /// Add a new entry to the constant pool for the current section and return an /// MCExpr that can be used to refer to the constant pool location. diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h index 1b0e2e36bd5e..3e69c3e6e5d4 100644 --- a/include/llvm/Object/COFF.h +++ b/include/llvm/Object/COFF.h @@ -858,6 +858,9 @@ public: std::error_code getExportRVA(uint32_t &Result) const; std::error_code getSymbolName(StringRef &Result) const; + std::error_code isForwarder(bool &Result) const; + std::error_code getForwardTo(StringRef &Result) const; + private: const export_directory_table_entry *ExportTable; uint32_t Index; diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h index 5823848aaacb..5d826da4c2fc 100644 --- a/include/llvm/Object/ELFObjectFile.h +++ b/include/llvm/Object/ELFObjectFile.h @@ -842,6 +842,8 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { case ELF::EM_SPARC: case ELF::EM_SPARC32PLUS: return "ELF32-sparc"; + case ELF::EM_WEBASSEMBLY: + return "ELF32-wasm"; default: return "ELF32-unknown"; } @@ -861,6 +863,8 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { return "ELF64-sparc"; case ELF::EM_MIPS: return "ELF64-mips"; + case ELF::EM_WEBASSEMBLY: + return "ELF64-wasm"; default: return "ELF64-unknown"; } @@ -908,6 +912,12 @@ unsigned ELFObjectFile<ELFT>::getArch() const { return IsLittleEndian ? Triple::sparcel : Triple::sparc; case ELF::EM_SPARCV9: return Triple::sparcv9; + case ELF::EM_WEBASSEMBLY: + switch (EF.getHeader()->e_ident[ELF::EI_CLASS]) { + case ELF::ELFCLASS32: return Triple::wasm32; + case ELF::ELFCLASS64: return Triple::wasm64; + default: return Triple::UnknownArch; + } default: return Triple::UnknownArch; diff --git a/include/llvm/Pass.h b/include/llvm/Pass.h index 3c4d838a4652..99604cdbc9ca 100644 --- a/include/llvm/Pass.h +++ b/include/llvm/Pass.h @@ -369,6 +369,10 @@ protected: /// @brief This is the storage for the -time-passes option. extern bool TimePassesIsEnabled; +/// isFunctionInPrintList - returns true if a function should be printed via +// debugging options like -print-after-all/-print-before-all. +// @brief Tells if the function IR should be printed by PrinterPass. +extern bool isFunctionInPrintList(StringRef FunctionName); } // End llvm namespace // Include support files that contain important APIs commonly used by Passes, diff --git a/include/llvm/ProfileData/CoverageMapping.h b/include/llvm/ProfileData/CoverageMapping.h index 3790e1358449..92a991eb39ba 100644 --- a/include/llvm/ProfileData/CoverageMapping.h +++ b/include/llvm/ProfileData/CoverageMapping.h @@ -20,13 +20,34 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/raw_ostream.h" #include <system_error> #include <tuple> namespace llvm { +namespace coverage { +enum class coveragemap_error { + success = 0, + eof, + no_data_found, + unsupported_version, + truncated, + malformed +}; +} // end of coverage namespace. +} + +namespace std { +template <> +struct is_error_code_enum<llvm::coverage::coveragemap_error> : std::true_type { +}; +} + +namespace llvm { class IndexedInstrProfReader; namespace coverage { @@ -35,8 +56,6 @@ class CoverageMappingReader; class CoverageMapping; struct CounterExpressions; -enum CoverageMappingVersion { CoverageMappingVersion1 }; - /// \brief A Counter is an abstract value that describes how to compute the /// execution count for a region of code using the collected profile count data. struct Counter { @@ -454,6 +473,76 @@ public: CoverageData getCoverageForExpansion(const ExpansionRecord &Expansion); }; +const std::error_category &coveragemap_category(); + +inline std::error_code make_error_code(coveragemap_error E) { + return std::error_code(static_cast<int>(E), coveragemap_category()); +} + +// Profile coverage map has the following layout: +// [CoverageMapFileHeader] +// [ArrayStart] +// [CovMapFunctionRecord] +// [CovMapFunctionRecord] +// ... +// [ArrayEnd] +// [Encoded Region Mapping Data] +LLVM_PACKED_START +template <class IntPtrT> struct CovMapFunctionRecord { +#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" + + // Return the structural hash associated with the function. + template <support::endianness Endian> uint64_t getFuncHash() const { + return support::endian::byte_swap<uint64_t, Endian>(FuncHash); + } + // Return the coverage map data size for the funciton. + template <support::endianness Endian> uint32_t getDataSize() const { + return support::endian::byte_swap<uint32_t, Endian>(DataSize); + } + // Return function lookup key. The value is consider opaque. + template <support::endianness Endian> IntPtrT getFuncNameRef() const { + return support::endian::byte_swap<IntPtrT, Endian>(NamePtr); + } + // Return the PGO name of the function */ + template <support::endianness Endian> + std::error_code getFuncName(InstrProfSymtab &ProfileNames, + StringRef &FuncName) const { + IntPtrT NameRef = getFuncNameRef<Endian>(); + uint32_t NameS = support::endian::byte_swap<uint32_t, Endian>(NameSize); + FuncName = ProfileNames.getFuncName(NameRef, NameS); + if (NameS && FuncName.empty()) + return coveragemap_error::malformed; + return std::error_code(); + } +}; +// Per module coverage mapping data header, i.e. CoverageMapFileHeader +// documented above. +struct CovMapHeader { +#define COVMAP_HEADER(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" + template <support::endianness Endian> uint32_t getNRecords() const { + return support::endian::byte_swap<uint32_t, Endian>(NRecords); + } + template <support::endianness Endian> uint32_t getFilenamesSize() const { + return support::endian::byte_swap<uint32_t, Endian>(FilenamesSize); + } + template <support::endianness Endian> uint32_t getCoverageSize() const { + return support::endian::byte_swap<uint32_t, Endian>(CoverageSize); + } + template <support::endianness Endian> uint32_t getVersion() const { + return support::endian::byte_swap<uint32_t, Endian>(Version); + } +}; + +LLVM_PACKED_END + +enum CoverageMappingVersion { + CoverageMappingVersion1 = 0, + // The current versin is Version1 + CoverageMappingCurrentVersion = INSTR_PROF_COVMAP_VERSION +}; + } // end namespace coverage /// \brief Provide DenseMapInfo for CounterExpression @@ -484,26 +573,6 @@ template<> struct DenseMapInfo<coverage::CounterExpression> { } }; -const std::error_category &coveragemap_category(); - -enum class coveragemap_error { - success = 0, - eof, - no_data_found, - unsupported_version, - truncated, - malformed -}; - -inline std::error_code make_error_code(coveragemap_error E) { - return std::error_code(static_cast<int>(E), coveragemap_category()); -} - } // end namespace llvm -namespace std { -template <> -struct is_error_code_enum<llvm::coveragemap_error> : std::true_type {}; -} - #endif // LLVM_PROFILEDATA_COVERAGEMAPPING_H_ diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 49569d89507b..c84d8d24f206 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -30,7 +30,6 @@ #include <system_error> #include <vector> -#define INSTR_PROF_INDEX_VERSION 3 namespace llvm { class Function; @@ -66,7 +65,8 @@ inline StringRef getInstrProfValueProfFuncName() { /// Return the name of the section containing function coverage mapping /// data. inline StringRef getInstrProfCoverageSectionName(bool AddSegment) { - return AddSegment ? "__DATA,__llvm_covmap" : "__llvm_covmap"; + return AddSegment ? "__DATA," INSTR_PROF_COVMAP_SECT_NAME_STR + : INSTR_PROF_COVMAP_SECT_NAME_STR; } /// Return the name prefix of variables containing instrumented function names. @@ -89,6 +89,12 @@ inline StringRef getCoverageMappingVarName() { return "__llvm_coverage_mapping"; } +/// Return the name of the internal variable recording the array +/// of PGO name vars referenced by the coverage mapping, The owning +/// functions of those names are not emitted by FE (e.g, unused inline +/// functions.) +inline StringRef getCoverageNamesVarName() { return "__llvm_coverage_names"; } + /// Return the name of function that registers all the per-function control /// data at program startup time by calling __llvm_register_function. This /// function has internal linkage and is called by __llvm_profile_init @@ -349,11 +355,14 @@ struct InstrProfValueSiteRecord { return left.Value < right.Value; }); } + /// Sort ValueData Descending by Count + inline void sortByCount(); /// Merge data from another InstrProfValueSiteRecord /// Optionally scale merged counts by \p Weight. - instrprof_error mergeValueData(InstrProfValueSiteRecord &Input, - uint64_t Weight = 1); + instrprof_error merge(InstrProfValueSiteRecord &Input, uint64_t Weight = 1); + /// Scale up value profile data counts. + instrprof_error scale(uint64_t Weight); }; /// Profiling information for a single function. @@ -396,6 +405,19 @@ struct InstrProfRecord { /// Optionally scale merged counts by \p Weight. instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1); + /// Scale up profile counts (including value profile data) by + /// \p Weight. + instrprof_error scale(uint64_t Weight); + + /// Sort value profile data (per site) by count. + void sortValueData() { + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { + std::vector<InstrProfValueSiteRecord> &SiteRecords = + getValueSitesForKind(Kind); + for (auto &SR : SiteRecords) + SR.sortByCount(); + } + } /// Clear value data entries void clearValueData() { for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) @@ -430,6 +452,8 @@ private: // Scale merged value counts by \p Weight. instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight); + // Scale up value profile data count. + instrprof_error scaleValueProfData(uint32_t ValueKind, uint64_t Weight); }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -497,11 +521,22 @@ inline support::endianness getHostEndianness() { #define INSTR_PROF_VALUE_PROF_DATA #include "llvm/ProfileData/InstrProfData.inc" - /* - * Initialize the record for runtime value profile data. - * Return 0 if the initialization is successful, otherwise - * return 1. - */ +void InstrProfValueSiteRecord::sortByCount() { + ValueData.sort( + [](const InstrProfValueData &left, const InstrProfValueData &right) { + return left.Count > right.Count; + }); + // Now truncate + size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE; + if (ValueData.size() > max_s) + ValueData.resize(max_s); +} + +/* +* Initialize the record for runtime value profile data. +* Return 0 if the initialization is successful, otherwise +* return 1. +*/ int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, const uint16_t *NumValueSites, ValueProfNode **Nodes); @@ -597,31 +632,6 @@ struct Header { } // end namespace RawInstrProf -namespace coverage { - -// Profile coverage map has the following layout: -// [CoverageMapFileHeader] -// [ArrayStart] -// [CovMapFunctionRecord] -// [CovMapFunctionRecord] -// ... -// [ArrayEnd] -// [Encoded Region Mapping Data] -LLVM_PACKED_START -template <class IntPtrT> struct CovMapFunctionRecord { - #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name; - #include "llvm/ProfileData/InstrProfData.inc" -}; -// Per module coverage mapping data header, i.e. CoverageMapFileHeader -// documented above. -struct CovMapHeader { -#define COVMAP_HEADER(Type, LLVMType, Name, Init) Type Name; -#include "llvm/ProfileData/InstrProfData.inc" -}; - -LLVM_PACKED_END -} - } // end namespace llvm namespace std { diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc index 3a7c0c5f2773..33c7d94aea2a 100644 --- a/include/llvm/ProfileData/InstrProfData.inc +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -28,7 +28,7 @@ * * Examples of how the template is used to instantiate structure definition: * 1. To declare a structure: - * + * * struct ProfData { * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ * Type Name; @@ -155,7 +155,7 @@ VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) #endif COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ - llvm::Type::getInt8PtrTy(Ctx))) + llvm::Type::getInt8PtrTy(Ctx))) COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ NameValue.size())) @@ -182,7 +182,7 @@ COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) COVMAP_HEADER(uint32_t, Int32Ty, Version, \ - llvm::ConstantInt::get(Int32Ty, CoverageMappingVersion1)) + llvm::ConstantInt::get(Int32Ty, CoverageMappingCurrentVersion)) #undef COVMAP_HEADER /* COVMAP_HEADER end. */ @@ -190,7 +190,8 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #ifdef INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_DATA_DEFINED -/*! +#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 +/*! * This is the header of the data structure that defines the on-disk * layout of the value profile data of a particular kind for one function. */ @@ -202,7 +203,7 @@ typedef struct ValueProfRecord { * otherwise the record for this kind won't be emitted. */ uint32_t NumValueSites; - /* + /* * The first element of the array that stores the number of profiled * values for each value site. The size of the array is NumValueSites. * Since NumValueSites is greater than zero, there is at least one @@ -226,7 +227,7 @@ typedef struct ValueProfRecord { * \brief Return the number of value sites. */ uint32_t getNumValueSites() const { return NumValueSites; } - /*! + /*! * \brief Read data from this record and save it to Record. */ void deserializeTo(InstrProfRecord &Record, @@ -247,10 +248,10 @@ typedef struct ValueProfRecord { typedef struct ValueProfData { /* * Total size in bytes including this field. It must be a multiple - * of sizeof(uint64_t). + * of sizeof(uint64_t). */ uint32_t TotalSize; - /* + /* *The number of value profile kinds that has value profile data. * In this implementation, a value profile kind is considered to * have profile data if the number of value profile sites for the @@ -260,7 +261,7 @@ typedef struct ValueProfData { */ uint32_t NumValueKinds; - /* + /* * Following are a sequence of variable length records. The prefix/header * of each record is defined by ValueProfRecord type. The number of * records is NumValueKinds. @@ -314,7 +315,7 @@ typedef struct ValueProfData { #endif } ValueProfData; -/* +/* * The closure is designed to abstact away two types of value profile data: * - InstrProfRecord which is the primary data structure used to * represent profile data in host tools (reader, writer, and profile-use) @@ -335,7 +336,7 @@ typedef struct ValueProfRecordClosure { uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); - /* + /* * After extracting the value profile data from the value profile record, * this method is used to map the in-memory value to on-disk value. If * the method is null, value will be written out untranslated. @@ -346,7 +347,7 @@ typedef struct ValueProfRecordClosure { ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); } ValueProfRecordClosure; -/* +/* * A wrapper struct that represents value profile runtime data. * Like InstrProfRecord class which is used by profiling host tools, * ValueProfRuntimeRecord also implements the abstract intefaces defined in @@ -384,7 +385,7 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, uint32_t getNumValueKindsRT(const void *R); #undef INSTR_PROF_VALUE_PROF_DATA -#endif /* INSTR_PROF_VALUE_PROF_DATA */ +#endif /* INSTR_PROF_VALUE_PROF_DATA */ #ifdef INSTR_PROF_COMMON_API_IMPL @@ -412,7 +413,7 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { return Size; } -/*! +/*! * \brief Return the total size of the value profile record including the * header and the value data. */ @@ -432,7 +433,7 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { This->NumValueSites)); } -/*! +/*! * \brief Return the total number of value data for \c This record. */ INSTR_PROF_INLINE @@ -444,7 +445,7 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { return NumValueData; } -/*! +/*! * \brief Use this method to advance to the next \c This \c ValueProfRecord. */ INSTR_PROF_INLINE @@ -465,7 +466,7 @@ ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { /* Closure based interfaces. */ -/*! +/*! * Return the total size in bytes of the on-disk value profile data * given the data stored in Record. */ @@ -535,7 +536,7 @@ ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, return VPD; } -/* +/* * The value profiler runtime library stores the value profile data * for a given function in \c NumValueSites and \c Nodes structures. * \c ValueProfRuntimeRecord class is used to encapsulate the runtime @@ -639,7 +640,7 @@ static ValueProfRecordClosure RTRecordClosure = {0, getValueForSiteRT, allocValueProfDataRT}; -/* +/* * Return the size of ValueProfData structure to store data * recorded in the runtime record. */ @@ -648,7 +649,7 @@ uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { return getValueProfDataSize(&RTRecordClosure); } -/* +/* * Return a ValueProfData instance that stores the data collected * from runtime. If \c DstData is provided by the caller, the value * profile data will be store in *DstData and DstData is returned, @@ -696,18 +697,31 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, /* Raw profile format version. */ #define INSTR_PROF_RAW_VERSION 2 +#define INSTR_PROF_INDEX_VERSION 3 +#define INSTR_PROF_COVMAP_VERSION 0 + +/* Profile version is always of type uint_64_t. Reserve the upper 8 bits in the + * version for other variants of profile. We set the lowest bit of the upper 8 + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * generated profile, and 0 if this is a Clang FE generated profile. +*/ +#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names #define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts +#define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) +#define INSTR_PROF_DATA_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) +#define INSTR_PROF_NAME_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) +#define INSTR_PROF_CNTS_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) +#define INSTR_PROF_COVMAP_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -751,4 +765,3 @@ typedef struct ValueProfNode { #else #undef INSTR_PROF_DATA_DEFINED #endif - diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 8df3fe803209..6c39cf9458dc 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -140,16 +140,9 @@ public: /// around unsigned integers. sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) { bool Overflowed; - if (Weight > 1) { - S = SaturatingMultiply(S, Weight, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - } - NumSamples = SaturatingAdd(NumSamples, S, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - - return sampleprof_error::success; + NumSamples = SaturatingMultiplyAdd(S, Weight, NumSamples, &Overflowed); + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; } /// Add called function \p F with samples \p S. @@ -161,16 +154,10 @@ public: uint64_t Weight = 1) { uint64_t &TargetSamples = CallTargets[F]; bool Overflowed; - if (Weight > 1) { - S = SaturatingMultiply(S, Weight, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - } - TargetSamples = SaturatingAdd(TargetSamples, S, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - - return sampleprof_error::success; + TargetSamples = + SaturatingMultiplyAdd(S, Weight, TargetSamples, &Overflowed); + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; } /// Return true if this sample record contains function calls. @@ -215,29 +202,17 @@ public: void dump() const; sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) { bool Overflowed; - if (Weight > 1) { - Num = SaturatingMultiply(Num, Weight, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - } - TotalSamples = SaturatingAdd(TotalSamples, Num, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - - return sampleprof_error::success; + TotalSamples = + SaturatingMultiplyAdd(Num, Weight, TotalSamples, &Overflowed); + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; } sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) { bool Overflowed; - if (Weight > 1) { - Num = SaturatingMultiply(Num, Weight, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - } - TotalHeadSamples = SaturatingAdd(TotalHeadSamples, Num, &Overflowed); - if (Overflowed) - return sampleprof_error::counter_overflow; - - return sampleprof_error::success; + TotalHeadSamples = + SaturatingMultiplyAdd(Num, Weight, TotalHeadSamples, &Overflowed); + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; } sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num, uint64_t Weight = 1) { diff --git a/include/llvm/Support/ARMTargetParser.def b/include/llvm/Support/ARMTargetParser.def index c895b095bc5b..d94a51b8bcf9 100644 --- a/include/llvm/Support/ARMTargetParser.def +++ b/include/llvm/Support/ARMTargetParser.def @@ -96,7 +96,7 @@ ARM_ARCH("iwmmxt", AK_IWMMXT, "iwmmxt", "", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, AEK_NONE) ARM_ARCH("iwmmxt2", AK_IWMMXT2, "iwmmxt2", "", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, AEK_NONE) -ARM_ARCH("xscale", AK_XSCALE, "xscale", "", ARMBuildAttrs::CPUArch::v5TE, +ARM_ARCH("xscale", AK_XSCALE, "xscale", "v5e", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, AEK_NONE) ARM_ARCH("armv7s", AK_ARMV7S, "7-S", "v7s", ARMBuildAttrs::CPUArch::v7, FK_NEON_VFPV4, AEK_DSP) diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h index c608736fa956..043d82314609 100644 --- a/include/llvm/Support/Allocator.h +++ b/include/llvm/Support/Allocator.h @@ -187,6 +187,7 @@ public: /// \brief Deallocate all but the current slab and reset the current pointer /// to the beginning of it, freeing all memory allocated so far. void Reset() { + // Deallocate all but the first slab, and deallocate all custom-sized slabs. DeallocateCustomSizedSlabs(); CustomSizedSlabs.clear(); @@ -198,7 +199,7 @@ public: CurPtr = (char *)Slabs.front(); End = CurPtr + SlabSize; - // Deallocate all but the first slab, and deallocate all custom-sized slabs. + __asan_poison_memory_region(*Slabs.begin(), computeSlabSize(0)); DeallocateSlabs(std::next(Slabs.begin()), Slabs.end()); Slabs.erase(std::next(Slabs.begin()), Slabs.end()); } diff --git a/include/llvm/Support/COFF.h b/include/llvm/Support/COFF.h index 0162175efe3e..0245632c96a0 100644 --- a/include/llvm/Support/COFF.h +++ b/include/llvm/Support/COFF.h @@ -656,6 +656,15 @@ namespace COFF { } }; + enum CodeViewLine : unsigned { + CVL_LineNumberStartBits = 24, + CVL_LineNumberEndDeltaBits = 7, + CVL_LineNumberEndDeltaMask = (1U << CVL_LineNumberEndDeltaBits) - 1, + CVL_MaxLineNumber = (1U << CVL_LineNumberStartBits) - 1, + CVL_IsStatement = 1U << 31, + CVL_MaxColumnNumber = UINT16_MAX, + }; + enum CodeViewIdentifiers { DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS = 0x1, DEBUG_SECTION_MAGIC = 0x4, diff --git a/include/llvm/Support/ELF.h b/include/llvm/Support/ELF.h index 97708a7cdd63..e24420fc1fe4 100644 --- a/include/llvm/Support/ELF.h +++ b/include/llvm/Support/ELF.h @@ -309,7 +309,12 @@ enum { EM_COOL = 217, // iCelero CoolEngine EM_NORC = 218, // Nanoradio Optimized RISC EM_CSR_KALIMBA = 219, // CSR Kalimba architecture family - EM_AMDGPU = 224 // AMD GPU architecture + EM_AMDGPU = 224, // AMD GPU architecture + + // A request has been made to the maintainer of the official registry for + // such numbers for an official value for WebAssembly. As soon as one is + // allocated, this enum will be updated to use it. + EM_WEBASSEMBLY = 0x4157, // WebAssembly architecture }; // Object file classes. @@ -594,6 +599,11 @@ enum { #include "ELFRelocs/Sparc.def" }; +// ELF Relocation types for WebAssembly +enum { +#include "ELFRelocs/WebAssembly.def" +}; + #undef ELF_RELOC // Section header. @@ -1024,7 +1034,10 @@ enum { PT_AMDGPU_HSA_LOAD_GLOBAL_PROGRAM = 0x60000000, PT_AMDGPU_HSA_LOAD_GLOBAL_AGENT = 0x60000001, PT_AMDGPU_HSA_LOAD_READONLY_AGENT = 0x60000002, - PT_AMDGPU_HSA_LOAD_CODE_AGENT = 0x60000003 + PT_AMDGPU_HSA_LOAD_CODE_AGENT = 0x60000003, + + // WebAssembly program header types. + PT_WEBASSEMBLY_FUNCTIONS = PT_LOPROC + 0, // Function definitions. }; // Segment flag bits. diff --git a/include/llvm/Support/ELFRelocs/WebAssembly.def b/include/llvm/Support/ELFRelocs/WebAssembly.def new file mode 100644 index 000000000000..9a34349efb96 --- /dev/null +++ b/include/llvm/Support/ELFRelocs/WebAssembly.def @@ -0,0 +1,8 @@ + +#ifndef ELF_RELOC +#error "ELF_RELOC must be defined" +#endif + +ELF_RELOC(R_WEBASSEMBLY_NONE, 0) +ELF_RELOC(R_WEBASSEMBLY_DATA, 1) +ELF_RELOC(R_WEBASSEMBLY_FUNCTION, 2) diff --git a/include/llvm/Support/GenericDomTree.h b/include/llvm/Support/GenericDomTree.h index 8751f272cd29..8bae582d18c7 100644 --- a/include/llvm/Support/GenericDomTree.h +++ b/include/llvm/Support/GenericDomTree.h @@ -724,25 +724,17 @@ public: if (!this->IsPostDominators) { // Initialize root NodeT *entry = TraitsTy::getEntryNode(&F); - this->Roots.push_back(entry); - this->IDoms[entry] = nullptr; - this->DomTreeNodes[entry] = nullptr; + addRoot(entry); Calculate<FT, NodeT *>(*this, F); } else { // Initialize the roots list for (typename TraitsTy::nodes_iterator I = TraitsTy::nodes_begin(&F), E = TraitsTy::nodes_end(&F); - I != E; ++I) { + I != E; ++I) if (TraitsTy::child_begin(&*I) == TraitsTy::child_end(&*I)) addRoot(&*I); - // Prepopulate maps so that we don't get iterator invalidation issues - // later. - this->IDoms[&*I] = nullptr; - this->DomTreeNodes[&*I] = nullptr; - } - Calculate<FT, Inverse<NodeT *>>(*this, F); } } diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h index 8111aeebe6ee..408ae3c339a2 100644 --- a/include/llvm/Support/MathExtras.h +++ b/include/llvm/Support/MathExtras.h @@ -717,6 +717,25 @@ SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) { return Z; } +/// \brief Multiply two unsigned integers, X and Y, and add the unsigned +/// integer, A to the product. Clamp the result to the maximum representable +/// value of T on overflow. ResultOverflowed indicates if the result is larger +/// than the maximum representable value of type T. +/// Note that this is purely a convenience function as there is no distinction +/// where overflow occurred in a 'fused' multiply-add for unsigned numbers. +template <typename T> +typename std::enable_if<std::is_unsigned<T>::value, T>::type +SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) { + bool Dummy; + bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; + + T Product = SaturatingMultiply(X, Y, &Overflowed); + if (Overflowed) + return Product; + + return SaturatingAdd(A, Product, &Overflowed); +} + extern const float huge_valf; } // End llvm namespace diff --git a/include/llvm/Transforms/IPO.h b/include/llvm/Transforms/IPO.h index 0c374a070ce8..78d2fadc5190 100644 --- a/include/llvm/Transforms/IPO.h +++ b/include/llvm/Transforms/IPO.h @@ -183,12 +183,20 @@ ModulePass *createBlockExtractorPass(); ModulePass *createStripDeadPrototypesPass(); //===----------------------------------------------------------------------===// -/// createFunctionAttrsPass - This pass discovers functions that do not access -/// memory, or only read memory, and gives them the readnone/readonly attribute. -/// It also discovers function arguments that are not captured by the function -/// and marks them with the nocapture attribute. +/// createPostOrderFunctionAttrsPass - This pass walks SCCs of the call graph +/// in post-order to deduce and propagate function attributes. It can discover +/// functions that do not access memory, or only read memory, and give them the +/// readnone/readonly attribute. It also discovers function arguments that are +/// not captured by the function and marks them with the nocapture attribute. /// -Pass *createFunctionAttrsPass(); +Pass *createPostOrderFunctionAttrsPass(); + +//===----------------------------------------------------------------------===// +/// createReversePostOrderFunctionAttrsPass - This pass walks SCCs of the call +/// graph in RPO to deduce and propagate function attributes. Currently it +/// only handles synthesizing norecurse attributes. +/// +Pass *createReversePostOrderFunctionAttrsPass(); //===----------------------------------------------------------------------===// /// createMergeFunctionsPass - This pass discovers identical functions and diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h index 92a1d52f1011..4f006f2adeef 100644 --- a/include/llvm/Transforms/Utils/Cloning.h +++ b/include/llvm/Transforms/Utils/Cloning.h @@ -147,42 +147,12 @@ void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueMapTypeRemapper *TypeMapper = nullptr, ValueMaterializer *Materializer = nullptr); -/// A helper class used with CloneAndPruneIntoFromInst to change the default -/// behavior while instructions are being cloned. -class CloningDirector { -public: - /// This enumeration describes the way CloneAndPruneIntoFromInst should - /// proceed after the CloningDirector has examined an instruction. - enum CloningAction { - ///< Continue cloning the instruction (default behavior). - CloneInstruction, - ///< Skip this instruction but continue cloning the current basic block. - SkipInstruction, - ///< Skip this instruction and stop cloning the current basic block. - StopCloningBB, - ///< Don't clone the terminator but clone the current block's successors. - CloneSuccessors - }; - - virtual ~CloningDirector() {} - - /// Subclasses must override this function to customize cloning behavior. - virtual CloningAction handleInstruction(ValueToValueMapTy &VMap, - const Instruction *Inst, - BasicBlock *NewBB) = 0; - - virtual ValueMapTypeRemapper *getTypeRemapper() { return nullptr; } - virtual ValueMaterializer *getValueMaterializer() { return nullptr; } -}; - void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst, ValueToValueMapTy &VMap, bool ModuleLevelChanges, - SmallVectorImpl<ReturnInst*> &Returns, - const char *NameSuffix = "", - ClonedCodeInfo *CodeInfo = nullptr, - CloningDirector *Director = nullptr); - + SmallVectorImpl<ReturnInst *> &Returns, + const char *NameSuffix = "", + ClonedCodeInfo *CodeInfo = nullptr); /// CloneAndPruneFunctionInto - This works exactly like CloneFunctionInto, /// except that it does some simple constant prop and DCE on the fly. The diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h index 81b376f0c212..911c6f14da0b 100644 --- a/include/llvm/Transforms/Utils/Local.h +++ b/include/llvm/Transforms/Utils/Local.h @@ -42,6 +42,7 @@ class TargetLibraryInfo; class TargetTransformInfo; class DIBuilder; class DominatorTree; +class LazyValueInfo; template<typename T> class SmallVectorImpl; @@ -303,7 +304,7 @@ void removeUnwindEdge(BasicBlock *BB); /// \brief Remove all blocks that can not be reached from the function's entry. /// /// Returns true if any basic block was removed. -bool removeUnreachableBlocks(Function &F); +bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI = nullptr); /// \brief Combine the metadata of two instructions so that K can replace J /// diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap index 0adce0c9602d..d74ada6faa61 100644 --- a/include/llvm/module.modulemap +++ b/include/llvm/module.modulemap @@ -207,6 +207,7 @@ module LLVM_Utils { textual header "Support/ELFRelocs/Sparc.def" textual header "Support/ELFRelocs/SystemZ.def" textual header "Support/ELFRelocs/x86_64.def" + textual header "Support/ELFRelocs/WebAssembly.def" } // This part of the module is usable from both C and C++ code. |