diff options
Diffstat (limited to 'include/llvm/ExecutionEngine')
25 files changed, 1361 insertions, 763 deletions
diff --git a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h index 8d2f641254b3..72687682f606 100644 --- a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h +++ b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -22,17 +22,21 @@ namespace llvm { namespace jitlink { /// Registers all FDEs in the given eh-frame section with the current process. -Error registerEHFrameSection(const void *EHFrameSectionAddr); +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); /// Deregisters all FDEs in the given eh-frame section with the current process. -Error deregisterEHFrameSection(const void *EHFrameSectionAddr); +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); /// Supports registration/deregistration of EH-frames in a target process. class EHFrameRegistrar { public: virtual ~EHFrameRegistrar(); - virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; - virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; + virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) = 0; + virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) = 0; }; /// Registers / Deregisters EH-frames in the current process. @@ -48,31 +52,38 @@ public: InProcessEHFrameRegistrar(InProcessEHFrameRegistrar &&) = delete; InProcessEHFrameRegistrar &operator=(InProcessEHFrameRegistrar &&) = delete; - Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) override { + Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { return registerEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr)); + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); } - Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) override { + Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { return deregisterEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr)); + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); } private: InProcessEHFrameRegistrar(); }; -using StoreFrameAddressFunction = std::function<void(JITTargetAddress)>; +using StoreFrameRangeFunction = + std::function<void(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize)>; -/// Creates a pass that records the address of the EH frame section. If no -/// eh-frame section is found, it will set EHFrameAddr to zero. +/// Creates a pass that records the address and size of the EH frame section. +/// If no eh-frame section is found then the address and size will both be given +/// as zero. /// /// Authors of JITLinkContexts can use this function to register a post-fixup -/// pass that records the address of the eh-frame section. This address can +/// pass that records the range of the eh-frame section. This range can /// be used after finalization to register and deregister the frame. -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, - StoreFrameAddressFunction StoreFrameAddress); + StoreFrameRangeFunction StoreFrameRange); } // end namespace jitlink } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index be80d44ccf51..b531127cf892 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -34,6 +34,9 @@ namespace llvm { namespace jitlink { +class Symbol; +class Section; + /// Base class for errors originating in JIT linker, e.g. missing relocation /// support. class JITLinkError : public ErrorInfo<JITLinkError> { @@ -50,27 +53,22 @@ private: std::string ErrMsg; }; -// Forward declare the Atom class. -class Atom; - -/// Edge class. Represents both object file relocations, as well as layout and -/// keep-alive constraints. +/// Represents fixups and constraints in the LinkGraph. class Edge { public: using Kind = uint8_t; - using GenericEdgeKind = enum : Kind { + enum GenericEdgeKind : Kind { Invalid, // Invalid edge value. FirstKeepAlive, // Keeps target alive. Offset/addend zero. KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. - LayoutNext, // Layout constraint. Offset/Addend zero. FirstRelocation // First architecture specific relocation. }; using OffsetT = uint32_t; using AddendT = int64_t; - Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + Edge(Kind K, OffsetT Offset, Symbol &Target, AddendT Addend) : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} OffsetT getOffset() const { return Offset; } @@ -82,461 +80,637 @@ public: return K - FirstRelocation; } bool isKeepAlive() const { return K >= FirstKeepAlive; } - Atom &getTarget() const { return *Target; } - void setTarget(Atom &Target) { this->Target = &Target; } + Symbol &getTarget() const { return *Target; } + void setTarget(Symbol &Target) { this->Target = &Target; } AddendT getAddend() const { return Addend; } void setAddend(AddendT Addend) { this->Addend = Addend; } private: - Atom *Target; - OffsetT Offset; - AddendT Addend; + Symbol *Target = nullptr; + OffsetT Offset = 0; + AddendT Addend = 0; Kind K = 0; }; -using EdgeVector = std::vector<Edge>; +/// Returns the string name of the given generic edge kind, or "unknown" +/// otherwise. Useful for debugging. +const char *getGenericEdgeKindName(Edge::Kind K); -const StringRef getGenericEdgeKindName(Edge::Kind K); - -/// Base Atom class. Used by absolute and undefined atoms. -class Atom { - friend class AtomGraph; +/// Base class for Addressable entities (externals, absolutes, blocks). +class Addressable { + friend class LinkGraph; protected: - /// Create a named (as yet unresolved) atom. - Atom(StringRef Name) - : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false), - IsGlobal(false), IsAbsolute(false), IsCallable(false), - IsExported(false), IsWeak(false), HasLayoutNext(false), - IsCommon(false) {} - - /// Create an absolute symbol atom. - Atom(StringRef Name, JITTargetAddress Address) - : Name(Name), Address(Address), IsDefined(true), IsLive(false), - ShouldDiscard(false), IsGlobal(false), IsAbsolute(false), - IsCallable(false), IsExported(false), IsWeak(false), - HasLayoutNext(false), IsCommon(false) {} + Addressable(JITTargetAddress Address, bool IsDefined) + : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {} -public: - /// Returns true if this atom has a name. - bool hasName() const { return Name != StringRef(); } + Addressable(JITTargetAddress Address) + : Address(Address), IsDefined(false), IsAbsolute(true) { + assert(!(IsDefined && IsAbsolute) && + "Block cannot be both defined and absolute"); + } - /// Returns the name of this atom. - StringRef getName() const { return Name; } +public: + Addressable(const Addressable &) = delete; + Addressable &operator=(const Addressable &) = default; + Addressable(Addressable &&) = delete; + Addressable &operator=(Addressable &&) = default; - /// Returns the current target address of this atom. - /// The initial target address (for atoms that have one) will be taken from - /// the input object file's virtual address space. During the layout phase - /// of JIT linking the atom's address will be updated to point to its final - /// address in the JIT'd process. JITTargetAddress getAddress() const { return Address; } - - /// Set the current target address of this atom. void setAddress(JITTargetAddress Address) { this->Address = Address; } - /// Returns true if this is a defined atom. - bool isDefined() const { return IsDefined; } + /// Returns true if this is a defined addressable, in which case you + /// can downcast this to a . + bool isDefined() const { return static_cast<bool>(IsDefined); } + bool isAbsolute() const { return static_cast<bool>(IsAbsolute); } - /// Returns true if this atom is marked as live. - bool isLive() const { return IsLive; } +private: + JITTargetAddress Address = 0; + uint64_t IsDefined : 1; + uint64_t IsAbsolute : 1; +}; - /// Mark this atom as live. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setLive(bool IsLive) { - assert((IsDefined || IsAbsolute || !IsLive) && - "Only defined and absolute atoms can be marked live"); - this->IsLive = IsLive; - } +using BlockOrdinal = unsigned; +using SectionOrdinal = unsigned; - /// Returns true if this atom should be discarded during pruning. - bool shouldDiscard() const { return ShouldDiscard; } +/// An Addressable with content and edges. +class Block : public Addressable { + friend class LinkGraph; - /// Mark this atom to be discarded. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setShouldDiscard(bool ShouldDiscard) { - assert((IsDefined || IsAbsolute || !ShouldDiscard) && - "Only defined and absolute atoms can be marked live"); - this->ShouldDiscard = ShouldDiscard; +private: + /// Create a zero-fill defined addressable. + Block(Section &Parent, BlockOrdinal Ordinal, JITTargetAddress Size, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Size(Size), + Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; } - /// Returns true if this definition is global (i.e. visible outside this - /// linkage unit). - /// - /// Note: This is distict from Exported, which means visibile outside the - /// JITDylib that this graph is being linked in to. - bool isGlobal() const { return IsGlobal; } + /// Create a defined addressable for the given content. + Block(Section &Parent, BlockOrdinal Ordinal, StringRef Content, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Data(Content.data()), + Size(Content.size()), Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } - /// Mark this atom as global. - void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; } +public: + using EdgeVector = std::vector<Edge>; + using edge_iterator = EdgeVector::iterator; + using const_edge_iterator = EdgeVector::const_iterator; - /// Returns true if this atom represents an absolute symbol. - bool isAbsolute() const { return IsAbsolute; } + Block(const Block &) = delete; + Block &operator=(const Block &) = delete; + Block(Block &&) = delete; + Block &operator=(Block &&) = delete; - /// Returns true if this atom is known to be callable. + /// Return the parent section for this block. + Section &getSection() const { return Parent; } + + /// Return the ordinal for this block. + BlockOrdinal getOrdinal() const { return Ordinal; } + + /// Returns true if this is a zero-fill block. /// - /// Primarily provided for easy interoperability with ORC, which uses the - /// JITSymbolFlags::Common flag to identify symbols that can be interposed - /// with stubs. - bool isCallable() const { return IsCallable; } + /// If true, getSize is callable but getContent is not (the content is + /// defined to be a sequence of zero bytes of length Size). + bool isZeroFill() const { return !Data; } + + /// Returns the size of this defined addressable. + size_t getSize() const { return Size; } + + /// Get the content for this block. Block must not be a zero-fill block. + StringRef getContent() const { + assert(Data && "Section does not contain content"); + return StringRef(Data, Size); + } - /// Mark this atom as callable. - void setCallable(bool IsCallable) { - assert((IsDefined || IsAbsolute || !IsCallable) && - "Callable atoms must be defined or absolute"); - this->IsCallable = IsCallable; + /// Set the content for this block. + /// Caller is responsible for ensuring the underlying bytes are not + /// deallocated while pointed to by this block. + void setContent(StringRef Content) { + Data = Content.data(); + Size = Content.size(); } - /// Returns true if this atom should appear in the symbol table of a final - /// linked image. - bool isExported() const { return IsExported; } + /// Get the alignment for this content. + uint64_t getAlignment() const { return 1ull << P2Align; } + + /// Get the alignment offset for this content. + uint64_t getAlignmentOffset() const { return AlignmentOffset; } - /// Mark this atom as exported. - void setExported(bool IsExported) { - assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) && - "Exported atoms must have names"); - this->IsExported = IsExported; + /// Add an edge to this block. + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, + Edge::AddendT Addend) { + Edges.push_back(Edge(K, Offset, Target, Addend)); } - /// Returns true if this is a weak symbol. - bool isWeak() const { return IsWeak; } + /// Return the list of edges attached to this content. + iterator_range<edge_iterator> edges() { + return make_range(Edges.begin(), Edges.end()); + } - /// Mark this atom as weak. - void setWeak(bool IsWeak) { this->IsWeak = IsWeak; } + /// Returns the list of edges attached to this content. + iterator_range<const_edge_iterator> edges() const { + return make_range(Edges.begin(), Edges.end()); + } -private: - StringRef Name; - JITTargetAddress Address = 0; + /// Return the size of the edges list. + size_t edges_size() const { return Edges.size(); } - bool IsDefined : 1; - bool IsLive : 1; - bool ShouldDiscard : 1; + /// Returns true if the list of edges is empty. + bool edges_empty() const { return Edges.empty(); } - bool IsGlobal : 1; - bool IsAbsolute : 1; - bool IsCallable : 1; - bool IsExported : 1; - bool IsWeak : 1; +private: + static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1; -protected: - // These flags only make sense for DefinedAtom, but we can minimize the size - // of DefinedAtom by defining them here. - bool HasLayoutNext : 1; - bool IsCommon : 1; + uint64_t P2Align : 5; + uint64_t AlignmentOffset : 57; + Section &Parent; + const char *Data = nullptr; + size_t Size = 0; + BlockOrdinal Ordinal = 0; + std::vector<Edge> Edges; }; -// Forward declare DefinedAtom. -class DefinedAtom; +/// Describes symbol linkage. This can be used to make resolve definition +/// clashes. +enum class Linkage : uint8_t { + Strong, + Weak, +}; -raw_ostream &operator<<(raw_ostream &OS, const Atom &A); -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, - StringRef EdgeKindName); +/// For errors and debugging output. +const char *getLinkageName(Linkage L); + +/// Defines the scope in which this symbol should be visible: +/// Default -- Visible in the public interface of the linkage unit. +/// Hidden -- Visible within the linkage unit, but not exported from it. +/// Local -- Visible only within the LinkGraph. +enum class Scope : uint8_t { Default, Hidden, Local }; + +/// For debugging output. +const char *getScopeName(Scope S); + +raw_ostream &operator<<(raw_ostream &OS, const Block &B); + +/// Symbol representation. +/// +/// Symbols represent locations within Addressable objects. +/// They can be either Named or Anonymous. +/// Anonymous symbols have neither linkage nor visibility, and must point at +/// ContentBlocks. +/// Named symbols may be in one of four states: +/// - Null: Default initialized. Assignable, but otherwise unusable. +/// - Defined: Has both linkage and visibility and points to a ContentBlock +/// - Common: Has both linkage and visibility, points to a null Addressable. +/// - External: Has neither linkage nor visibility, points to an external +/// Addressable. +/// +class Symbol { + friend class LinkGraph; + +private: + Symbol(Addressable &Base, JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, bool IsLive, + bool IsCallable) + : Name(Name), Base(&Base), Offset(Offset), Size(Size) { + setLinkage(L); + setScope(S); + setLive(IsLive); + setCallable(IsCallable); + } + + static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name, + JITTargetAddress Size, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Common symbol name cannot be empty"); + assert(Base.isDefined() && + "Cannot create common symbol from undefined block"); + assert(static_cast<Block &>(Base).getSize() == Size && + "Common symbol size should match underlying block size"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Weak, S, IsLive, false); + return *Sym; + } + + static Symbol &constructExternal(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create external symbol from defined block"); + assert(!Name.empty() && "External symbol name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Strong, Scope::Default, + false, false); + return *Sym; + } + + static Symbol &constructAbsolute(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size, + Linkage L, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create absolute symbol from a defined block"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false); + return *Sym; + } + + static Symbol &constructAnonDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, + Scope::Local, IsLive, IsCallable); + return *Sym; + } + + static Symbol &constructNamedDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, + bool IsLive, bool IsCallable) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); + return *Sym; + } -/// Represents a section address range via a pair of DefinedAtom pointers to -/// the first and last atoms in the section. -class SectionRange { public: - SectionRange() = default; - SectionRange(DefinedAtom *First, DefinedAtom *Last) - : First(First), Last(Last) {} - DefinedAtom *getFirstAtom() const { - assert((!Last || First) && "First can not be null if end is non-null"); - return First; + /// Create a null Symbol. This allows Symbols to be default initialized for + /// use in containers (e.g. as map values). Null symbols are only useful for + /// assigning to. + Symbol() = default; + + // Symbols are not movable or copyable. + Symbol(const Symbol &) = delete; + Symbol &operator=(const Symbol &) = delete; + Symbol(Symbol &&) = delete; + Symbol &operator=(Symbol &&) = delete; + + /// Returns true if this symbol has a name. + bool hasName() const { return !Name.empty(); } + + /// Returns the name of this symbol (empty if the symbol is anonymous). + StringRef getName() const { + assert((!Name.empty() || getScope() == Scope::Local) && + "Anonymous symbol has non-local scope"); + return Name; } - DefinedAtom *getLastAtom() const { - assert((First || !Last) && "Last can not be null if start is non-null"); - return Last; + + /// Returns true if this Symbol has content (potentially) defined within this + /// object file (i.e. is anything but an external or absolute symbol). + bool isDefined() const { + assert(Base && "Attempt to access null symbol"); + return Base->isDefined(); } - bool isEmpty() const { - assert((First || !Last) && "Last can not be null if start is non-null"); - return !First; + + /// Returns true if this symbol is live (i.e. should be treated as a root for + /// dead stripping). + bool isLive() const { + assert(Base && "Attempting to access null symbol"); + return IsLive; } - JITTargetAddress getStart() const; - JITTargetAddress getEnd() const; - uint64_t getSize() const; -private: - DefinedAtom *First = nullptr; - DefinedAtom *Last = nullptr; -}; + /// Set this symbol's live bit. + void setLive(bool IsLive) { this->IsLive = IsLive; } -/// Represents an object file section. -class Section { - friend class AtomGraph; + /// Returns true is this symbol is callable. + bool isCallable() const { return IsCallable; } -private: - Section(StringRef Name, uint32_t Alignment, sys::Memory::ProtectionFlags Prot, - unsigned Ordinal, bool IsZeroFill) - : Name(Name), Alignment(Alignment), Prot(Prot), Ordinal(Ordinal), - IsZeroFill(IsZeroFill) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of 2"); + /// Set this symbol's callable bit. + void setCallable(bool IsCallable) { this->IsCallable = IsCallable; } + + /// Returns true if the underlying addressable is an unresolved external. + bool isExternal() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && !Base->isAbsolute(); } - using DefinedAtomSet = DenseSet<DefinedAtom *>; + /// Returns true if the underlying addressable is an absolute symbol. + bool isAbsolute() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && Base->isAbsolute(); + } -public: - using atom_iterator = DefinedAtomSet::iterator; - using const_atom_iterator = DefinedAtomSet::const_iterator; + /// Return the addressable that this symbol points to. + Addressable &getAddressable() { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } - ~Section(); - StringRef getName() const { return Name; } - uint32_t getAlignment() const { return Alignment; } - sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } - unsigned getSectionOrdinal() const { return Ordinal; } - size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } + /// Return the addressable that thsi symbol points to. + const Addressable &getAddressable() const { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } - bool isZeroFill() const { return IsZeroFill; } + /// Return the Block for this Symbol (Symbol must be defined). + Block &getBlock() { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<Block &>(*Base); + } - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range<atom_iterator> atoms() { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + /// Return the Block for this Symbol (Symbol must be defined). + const Block &getBlock() const { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<const Block &>(*Base); } - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range<const_atom_iterator> atoms() const { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + /// Returns the offset for this symbol within the underlying addressable. + JITTargetAddress getOffset() const { return Offset; } + + /// Returns the address of this symbol. + JITTargetAddress getAddress() const { return Base->getAddress() + Offset; } + + /// Returns the size of this symbol. + JITTargetAddress getSize() const { return Size; } + + /// Returns true if this symbol is backed by a zero-fill block. + /// This method may only be called on defined symbols. + bool isSymbolZeroFill() const { return getBlock().isZeroFill(); } + + /// Returns the content in the underlying block covered by this symbol. + /// This method may only be called on defined non-zero-fill symbols. + StringRef getSymbolContent() const { + return getBlock().getContent().substr(Offset, Size); } - /// Return the number of atoms in this section. - DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); } + /// Get the linkage for this Symbol. + Linkage getLinkage() const { return static_cast<Linkage>(L); } - /// Return true if this section contains no atoms. - bool atoms_empty() const { return DefinedAtoms.empty(); } + /// Set the linkage for this Symbol. + void setLinkage(Linkage L) { + assert((L == Linkage::Strong || (Base->isDefined() && !Name.empty())) && + "Linkage can only be applied to defined named symbols"); + this->L = static_cast<uint8_t>(L); + } - /// Returns the range of this section as the pair of atoms with the lowest - /// and highest target address. This operation is expensive, as it - /// must traverse all atoms in the section. - /// - /// Note: If the section is empty, both values will be null. The section - /// address will evaluate to null, and the size to zero. If the section - /// contains a single atom both values will point to it, the address will - /// evaluate to the address of that atom, and the size will be the size of - /// that atom. - SectionRange getRange() const; + /// Get the visibility for this Symbol. + Scope getScope() const { return static_cast<Scope>(S); } -private: - void addAtom(DefinedAtom &DA) { - assert(!DefinedAtoms.count(&DA) && "Atom is already in this section"); - DefinedAtoms.insert(&DA); + /// Set the visibility for this Symbol. + void setScope(Scope S) { + assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && + "Invalid visibility for symbol type"); + this->S = static_cast<uint8_t>(S); } - void removeAtom(DefinedAtom &DA) { - assert(DefinedAtoms.count(&DA) && "Atom is not in this section"); - DefinedAtoms.erase(&DA); +private: + void makeExternal(Addressable &A) { + assert(!A.isDefined() && "Attempting to make external with defined block"); + Base = &A; + Offset = 0; + setLinkage(Linkage::Strong); + setScope(Scope::Default); + IsLive = 0; + // note: Size and IsCallable fields left unchanged. } + static constexpr uint64_t MaxOffset = (1ULL << 59) - 1; + + // FIXME: A char* or SymbolStringPtr may pack better. StringRef Name; - uint32_t Alignment = 0; - sys::Memory::ProtectionFlags Prot; - unsigned Ordinal = 0; - unsigned NextAtomOrdinal = 0; - bool IsZeroFill = false; - DefinedAtomSet DefinedAtoms; + Addressable *Base = nullptr; + uint64_t Offset : 59; + uint64_t L : 1; + uint64_t S : 2; + uint64_t IsLive : 1; + uint64_t IsCallable : 1; + JITTargetAddress Size = 0; }; -/// Defined atom class. Suitable for use by defined named and anonymous -/// atoms. -class DefinedAtom : public Atom { - friend class AtomGraph; +raw_ostream &operator<<(raw_ostream &OS, const Symbol &A); + +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, + StringRef EdgeKindName); + +/// Represents an object file section. +class Section { + friend class LinkGraph; private: - DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) - : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), - Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } + Section(StringRef Name, sys::Memory::ProtectionFlags Prot, + SectionOrdinal SecOrdinal) + : Name(Name), Prot(Prot), SecOrdinal(SecOrdinal) {} - DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, - uint32_t Alignment) - : Atom(Name, Address), Parent(Parent), - Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } + using SymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; public: - using edge_iterator = EdgeVector::iterator; + using symbol_iterator = SymbolSet::iterator; + using const_symbol_iterator = SymbolSet::const_iterator; - Section &getSection() const { return Parent; } + using block_iterator = BlockSet::iterator; + using const_block_iterator = BlockSet::const_iterator; - uint64_t getSize() const { return Size; } + ~Section(); - StringRef getContent() const { - assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom"); - assert(Size <= std::numeric_limits<size_t>::max() && - "Content size too large"); - return {ContentPtr, static_cast<size_t>(Size)}; - } - void setContent(StringRef Content) { - assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?"); - ContentPtr = Content.data(); - Size = Content.size(); - } + /// Returns the name of this section. + StringRef getName() const { return Name; } + + /// Returns the protection flags for this section. + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } - bool isZeroFill() const { return Parent.isZeroFill(); } + /// Returns the ordinal for this section. + SectionOrdinal getOrdinal() const { return SecOrdinal; } - void setZeroFill(uint64_t Size) { - assert(Parent.isZeroFill() && !ContentPtr && - "Can't set zero-fill length of a non zero-fill atom"); - this->Size = Size; + /// Returns an iterator over the symbols defined in this section. + iterator_range<symbol_iterator> symbols() { + return make_range(Symbols.begin(), Symbols.end()); } - uint64_t getZeroFillSize() const { - assert(Parent.isZeroFill() && - "Can't get zero-fill length of a non zero-fill atom"); - return Size; + /// Returns an iterator over the symbols defined in this section. + iterator_range<const_symbol_iterator> symbols() const { + return make_range(Symbols.begin(), Symbols.end()); } - uint32_t getAlignment() const { return Alignment; } + /// Return the number of symbols in this section. + SymbolSet::size_type symbols_size() { return Symbols.size(); } - bool hasLayoutNext() const { return HasLayoutNext; } - void setLayoutNext(DefinedAtom &Next) { - assert(!HasLayoutNext && "Atom already has layout-next constraint"); - HasLayoutNext = true; - Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0)); - } - DefinedAtom &getLayoutNext() { - assert(HasLayoutNext && "Atom does not have a layout-next constraint"); - DefinedAtom *Next = nullptr; - for (auto &E : edges()) - if (E.getKind() == Edge::LayoutNext) { - assert(E.getTarget().isDefined() && - "layout-next target atom must be a defined atom"); - Next = static_cast<DefinedAtom *>(&E.getTarget()); - break; - } - assert(Next && "Missing LayoutNext edge"); - return *Next; - } + /// Return true if this section contains no symbols. + bool symbols_empty() const { return Symbols.empty(); } - bool isCommon() const { return IsCommon; } + /// Returns the ordinal for the next block. + BlockOrdinal getNextBlockOrdinal() { return NextBlockOrdinal++; } - void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target, - Edge::AddendT Addend) { - assert(K != Edge::LayoutNext && - "Layout edges should be added via setLayoutNext"); - Edges.push_back(Edge(K, Offset, Target, Addend)); +private: + void addSymbol(Symbol &Sym) { + assert(!Symbols.count(&Sym) && "Symbol is already in this section"); + Symbols.insert(&Sym); } - iterator_range<edge_iterator> edges() { - return make_range(Edges.begin(), Edges.end()); + void removeSymbol(Symbol &Sym) { + assert(Symbols.count(&Sym) && "symbol is not in this section"); + Symbols.erase(&Sym); } - size_t edges_size() const { return Edges.size(); } - bool edges_empty() const { return Edges.empty(); } - unsigned getOrdinal() const { return Ordinal; } + StringRef Name; + sys::Memory::ProtectionFlags Prot; + SectionOrdinal SecOrdinal = 0; + BlockOrdinal NextBlockOrdinal = 0; + SymbolSet Symbols; +}; -private: - void setCommon(uint64_t Size) { - assert(ContentPtr == 0 && "Atom already has content?"); - IsCommon = true; - setZeroFill(Size); +/// Represents a section address range via a pair of Block pointers +/// to the first and last Blocks in the section. +class SectionRange { +public: + SectionRange() = default; + SectionRange(const Section &Sec) { + if (Sec.symbols_empty()) + return; + First = Last = *Sec.symbols().begin(); + for (auto *Sym : Sec.symbols()) { + if (Sym->getAddress() < First->getAddress()) + First = Sym; + if (Sym->getAddress() > Last->getAddress()) + Last = Sym; + } + } + Symbol *getFirstSymbol() const { + assert((!Last || First) && "First can not be null if end is non-null"); + return First; + } + Symbol *getLastSymbol() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return Last; + } + bool isEmpty() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return !First; + } + JITTargetAddress getStart() const { + return First ? First->getBlock().getAddress() : 0; + } + JITTargetAddress getEnd() const { + return Last ? Last->getBlock().getAddress() + Last->getBlock().getSize() + : 0; } + uint64_t getSize() const { return getEnd() - getStart(); } - EdgeVector Edges; - uint64_t Size = 0; - Section &Parent; - const char *ContentPtr = nullptr; - unsigned Ordinal = 0; - uint32_t Alignment = 0; +private: + Symbol *First = nullptr; + Symbol *Last = nullptr; }; -inline JITTargetAddress SectionRange::getStart() const { - return First ? First->getAddress() : 0; -} +class LinkGraph { +private: + using SectionList = std::vector<std::unique_ptr<Section>>; + using ExternalSymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; + + template <typename... ArgTs> + Addressable &createAddressable(ArgTs &&... Args) { + Addressable *A = + reinterpret_cast<Addressable *>(Allocator.Allocate<Addressable>()); + new (A) Addressable(std::forward<ArgTs>(Args)...); + return *A; + } -inline JITTargetAddress SectionRange::getEnd() const { - return Last ? Last->getAddress() + Last->getSize() : 0; -} + void destroyAddressable(Addressable &A) { + A.~Addressable(); + Allocator.Deallocate(&A); + } -inline uint64_t SectionRange::getSize() const { return getEnd() - getStart(); } + template <typename... ArgTs> Block &createBlock(ArgTs &&... Args) { + Block *B = reinterpret_cast<Block *>(Allocator.Allocate<Block>()); + new (B) Block(std::forward<ArgTs>(Args)...); + Blocks.insert(B); + return *B; + } -inline SectionRange Section::getRange() const { - if (atoms_empty()) - return SectionRange(); - DefinedAtom *First = *DefinedAtoms.begin(), *Last = *DefinedAtoms.begin(); - for (auto *DA : atoms()) { - if (DA->getAddress() < First->getAddress()) - First = DA; - if (DA->getAddress() > Last->getAddress()) - Last = DA; + void destroyBlock(Block &B) { + Blocks.erase(&B); + B.~Block(); + Allocator.Deallocate(&B); } - return SectionRange(First, Last); -} -class AtomGraph { -private: - using SectionList = std::vector<std::unique_ptr<Section>>; - using AddressToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - using NamedAtomMap = DenseMap<StringRef, Atom *>; - using ExternalAtomSet = DenseSet<Atom *>; + void destroySymbol(Symbol &S) { + S.~Symbol(); + Allocator.Deallocate(&S); + } public: - using external_atom_iterator = ExternalAtomSet::iterator; + using external_symbol_iterator = ExternalSymbolSet::iterator; + + using block_iterator = BlockSet::iterator; using section_iterator = pointee_iterator<SectionList::iterator>; using const_section_iterator = pointee_iterator<SectionList::const_iterator>; - template <typename SecItrT, typename AtomItrT, typename T> - class defined_atom_iterator_impl + template <typename SectionItrT, typename SymbolItrT, typename T> + class defined_symbol_iterator_impl : public iterator_facade_base< - defined_atom_iterator_impl<SecItrT, AtomItrT, T>, + defined_symbol_iterator_impl<SectionItrT, SymbolItrT, T>, std::forward_iterator_tag, T> { public: - defined_atom_iterator_impl() = default; + defined_symbol_iterator_impl() = default; - defined_atom_iterator_impl(SecItrT SI, SecItrT SE) - : SI(SI), SE(SE), - AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) { - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl(SectionItrT SecI, SectionItrT SecE) + : SecI(SecI), SecE(SecE), + SymI(SecI != SecE ? SecI->symbols().begin() : SymbolItrT()) { + moveToNextSymbolOrEnd(); } - bool operator==(const defined_atom_iterator_impl &RHS) const { - return (SI == RHS.SI) && (AI == RHS.AI); + bool operator==(const defined_symbol_iterator_impl &RHS) const { + return (SecI == RHS.SecI) && (SymI == RHS.SymI); } T operator*() const { - assert(AI != SI->atoms().end() && "Dereferencing end?"); - return *AI; + assert(SymI != SecI->symbols().end() && "Dereferencing end?"); + return *SymI; } - defined_atom_iterator_impl operator++() { - ++AI; - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl operator++() { + ++SymI; + moveToNextSymbolOrEnd(); return *this; } private: - void moveToNextAtomOrEnd() { - while (SI != SE && AI == SI->atoms().end()) { - ++SI; - if (SI == SE) - AI = Section::atom_iterator(); - else - AI = SI->atoms().begin(); + void moveToNextSymbolOrEnd() { + while (SecI != SecE && SymI == SecI->symbols().end()) { + ++SecI; + SymI = SecI == SecE ? SymbolItrT() : SecI->symbols().begin(); } } - SecItrT SI, SE; - AtomItrT AI; + SectionItrT SecI, SecE; + SymbolItrT SymI; }; - using defined_atom_iterator = - defined_atom_iterator_impl<section_iterator, Section::atom_iterator, - DefinedAtom *>; + using defined_symbol_iterator = + defined_symbol_iterator_impl<const_section_iterator, + Section::symbol_iterator, Symbol *>; - using const_defined_atom_iterator = - defined_atom_iterator_impl<const_section_iterator, - Section::const_atom_iterator, - const DefinedAtom *>; + using const_defined_symbol_iterator = defined_symbol_iterator_impl< + const_section_iterator, Section::const_symbol_iterator, const Symbol *>; - AtomGraph(std::string Name, unsigned PointerSize, + LinkGraph(std::string Name, unsigned PointerSize, support::endianness Endianness) : Name(std::move(Name)), PointerSize(PointerSize), Endianness(Endianness) {} + ~LinkGraph(); + /// Returns the name of this graph (usually the name of the original /// underlying MemoryBuffer). const std::string &getName() { return Name; } @@ -544,84 +718,83 @@ public: /// Returns the pointer size for use in this graph. unsigned getPointerSize() const { return PointerSize; } - /// Returns the endianness of atom-content in this graph. + /// Returns the endianness of content in this graph. support::endianness getEndianness() const { return Endianness; } /// Create a section with the given name, protection flags, and alignment. - Section &createSection(StringRef Name, uint32_t Alignment, - sys::Memory::ProtectionFlags Prot, bool IsZeroFill) { - std::unique_ptr<Section> Sec( - new Section(Name, Alignment, Prot, Sections.size(), IsZeroFill)); + Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot) { + std::unique_ptr<Section> Sec(new Section(Name, Prot, Sections.size())); Sections.push_back(std::move(Sec)); return *Sections.back(); } - /// Add an external atom representing an undefined symbol in this graph. - Atom &addExternalAtom(StringRef Name) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast<Atom *>( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name); - ExternalAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a content block. + Block &createContentBlock(Section &Parent, StringRef Content, + uint64_t Address, uint64_t Alignment, + uint64_t AlignmentOffset) { + return createBlock(Parent, Parent.getNextBlockOrdinal(), Content, Address, + Alignment, AlignmentOffset); } - /// Add an external atom representing an absolute symbol. - Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast<Atom *>( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name, Addr); - AbsoluteAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a zero-fill block. + Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address, + uint64_t Alignment, uint64_t AlignmentOffset) { + return createBlock(Parent, Parent.getNextBlockOrdinal(), Size, Address, + Alignment, AlignmentOffset); } - /// Add an anonymous defined atom to the graph. - /// - /// Anonymous atoms have content but no name. They must have an address. - DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, - uint32_t Alignment) { - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - return *A; + /// Add an external symbol. + /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose + /// size is not known, you should substitute '0'. + Symbol &addExternalSymbol(StringRef Name, uint64_t Size) { + auto &Sym = Symbol::constructExternal( + Allocator.Allocate<Symbol>(), createAddressable(0, false), Name, Size); + ExternalSymbols.insert(&Sym); + return Sym; } - /// Add a defined atom to the graph. - /// - /// Allocates and constructs a DefinedAtom instance with the given parent, - /// name, address, and alignment. - DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - NamedAtoms[Name] = A; - return *A; + /// Add an absolute symbol. + Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address, + uint64_t Size, Linkage L, Scope S, bool IsLive) { + auto &Sym = Symbol::constructAbsolute(Allocator.Allocate<Symbol>(), + createAddressable(Address), Name, + Size, L, S, IsLive); + AbsoluteSymbols.insert(&Sym); + return Sym; } - /// Add a common symbol atom to the graph. - /// - /// Adds a common-symbol atom to the graph with the given parent, name, - /// address, alignment and size. - DefinedAtom &addCommonAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment, - uint64_t Size) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast<DefinedAtom *>( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - A->setCommon(Size); - Parent.addAtom(*A); - NamedAtoms[Name] = A; - return *A; + /// Convenience method for adding a weak zero-fill symbol. + Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section, + JITTargetAddress Address, uint64_t Size, + uint64_t Alignment, bool IsLive) { + auto &Sym = Symbol::constructCommon( + Allocator.Allocate<Symbol>(), + createBlock(Section, Section.getNextBlockOrdinal(), Address, Size, + Alignment, 0), + Name, Size, S, IsLive); + Section.addSymbol(Sym); + return Sym; + } + + /// Add an anonymous symbol. + Symbol &addAnonymousSymbol(Block &Content, JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + auto &Sym = Symbol::constructAnonDef(Allocator.Allocate<Symbol>(), Content, + Offset, Size, IsCallable, IsLive); + Content.getSection().addSymbol(Sym); + return Sym; + } + + /// Add a named symbol. + Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset, + StringRef Name, JITTargetAddress Size, Linkage L, + Scope S, bool IsCallable, bool IsLive) { + auto &Sym = + Symbol::constructNamedDef(Allocator.Allocate<Symbol>(), Content, Offset, + Name, Size, L, S, IsLive, IsCallable); + Content.getSection().addSymbol(Sym); + return Sym; } iterator_range<section_iterator> sections() { @@ -638,135 +811,79 @@ public: return nullptr; } - iterator_range<external_atom_iterator> external_atoms() { - return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + iterator_range<external_symbol_iterator> external_symbols() { + return make_range(ExternalSymbols.begin(), ExternalSymbols.end()); } - iterator_range<external_atom_iterator> absolute_atoms() { - return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end()); + iterator_range<external_symbol_iterator> absolute_symbols() { + return make_range(AbsoluteSymbols.begin(), AbsoluteSymbols.end()); } - iterator_range<defined_atom_iterator> defined_atoms() { - return make_range(defined_atom_iterator(Sections.begin(), Sections.end()), - defined_atom_iterator(Sections.end(), Sections.end())); + iterator_range<defined_symbol_iterator> defined_symbols() { + return make_range(defined_symbol_iterator(Sections.begin(), Sections.end()), + defined_symbol_iterator(Sections.end(), Sections.end())); } - iterator_range<const_defined_atom_iterator> defined_atoms() const { + iterator_range<const_defined_symbol_iterator> defined_symbols() const { return make_range( - const_defined_atom_iterator(Sections.begin(), Sections.end()), - const_defined_atom_iterator(Sections.end(), Sections.end())); - } - - /// Returns the atom with the given name, which must exist in this graph. - Atom &getAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); - return *I->second; - } - - /// Returns the atom with the given name, which must exist in this graph and - /// be a DefinedAtom. - DefinedAtom &getDefinedAtomByName(StringRef Name) { - auto &A = getAtomByName(Name); - assert(A.isDefined() && "Atom is not a defined atom"); - return static_cast<DefinedAtom &>(A); - } - - /// Search for the given atom by name. - /// Returns the atom (if found) or an error (if no atom with this name - /// exists). - Expected<Atom &> findAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error<JITLinkError>("No atom named " + Name); - return *I->second; - } - - /// Search for the given defined atom by name. - /// Returns the defined atom (if found) or an error (if no atom with this - /// name exists, or if one exists but is not a defined atom). - Expected<DefinedAtom &> findDefinedAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error<JITLinkError>("No atom named " + Name); - if (!I->second->isDefined()) - return make_error<JITLinkError>("Atom " + Name + - " exists but is not a " - "defined atom"); - return static_cast<DefinedAtom &>(*I->second); - } - - /// Returns the atom covering the given address, or an error if no such atom - /// exists. - /// - /// Returns null if no atom exists at the given address. - DefinedAtom *getAtomByAddress(JITTargetAddress Address) { - refreshAddrToAtomCache(); - - // If there are no defined atoms, bail out early. - if (AddrToAtomCache->empty()) - return nullptr; - - // Find the atom *after* the given address. - auto I = AddrToAtomCache->upper_bound(Address); - - // If this address falls before any known atom, bail out. - if (I == AddrToAtomCache->begin()) - return nullptr; - - // The atom we're looking for is the one before the atom we found. - --I; - - // Otherwise range check the atom that was found. - assert(!I->second->getContent().empty() && "Atom content not set"); - if (Address >= I->second->getAddress() + I->second->getContent().size()) - return nullptr; + const_defined_symbol_iterator(Sections.begin(), Sections.end()), + const_defined_symbol_iterator(Sections.end(), Sections.end())); + } - return I->second; + iterator_range<block_iterator> blocks() { + return make_range(Blocks.begin(), Blocks.end()); } - /// Like getAtomByAddress, but returns an Error if the given address is not - /// covered by an atom, rather than a null pointer. - Expected<DefinedAtom &> findAtomByAddress(JITTargetAddress Address) { - if (auto *DA = getAtomByAddress(Address)) - return *DA; - return make_error<JITLinkError>("No atom at address " + - formatv("{0:x16}", Address)); + /// Turn a defined symbol into an external one. + void makeExternal(Symbol &Sym) { + if (Sym.getAddressable().isAbsolute()) { + assert(AbsoluteSymbols.count(&Sym) && + "Sym is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + } else { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Section &Sec = Sym.getBlock().getSection(); + Sec.removeSymbol(Sym); + } + Sym.makeExternal(createAddressable(false)); + ExternalSymbols.insert(&Sym); } - // Remove the given external atom from the graph. - void removeExternalAtom(Atom &A) { - assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom"); - assert(ExternalAtoms.count(&A) && "A is not in the external atoms set"); - ExternalAtoms.erase(&A); - A.~Atom(); + /// Removes an external symbol. Also removes the underlying Addressable. + void removeExternalSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && !Sym.isAbsolute() && + "Sym is not an external symbol"); + assert(ExternalSymbols.count(&Sym) && "Symbol is not in the externals set"); + ExternalSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); } - /// Remove the given absolute atom from the graph. - void removeAbsoluteAtom(Atom &A) { - assert(A.isAbsolute() && "A is not an absolute atom"); - assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set"); - AbsoluteAtoms.erase(&A); - A.~Atom(); + /// Remove an absolute symbol. Also removes the underlying Addressable. + void removeAbsoluteSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && Sym.isAbsolute() && + "Sym is not an absolute symbol"); + assert(AbsoluteSymbols.count(&Sym) && + "Symbol is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); } - /// Remove the given defined atom from the graph. - void removeDefinedAtom(DefinedAtom &DA) { - if (AddrToAtomCache) { - assert(AddrToAtomCache->count(DA.getAddress()) && - "Cache exists, but does not contain atom"); - AddrToAtomCache->erase(DA.getAddress()); - } - if (DA.hasName()) { - assert(NamedAtoms.count(DA.getName()) && "Named atom not in map"); - NamedAtoms.erase(DA.getName()); - } - DA.getSection().removeAtom(DA); - DA.~DefinedAtom(); + /// Removes defined symbols. Does not remove the underlying block. + void removeDefinedSymbol(Symbol &Sym) { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Sym.getBlock().getSection().removeSymbol(Sym); + destroySymbol(Sym); } - /// Invalidate the atom-to-address map. - void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + /// Remove a block. + void removeBlock(Block &B) { + Blocks.erase(&B); + destroyBlock(B); + } /// Dump the graph. /// @@ -778,87 +895,84 @@ public: std::function<StringRef(Edge::Kind)>()); private: - AddressToAtomMap &getAddrToAtomMap() { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - const AddressToAtomMap &getAddrToAtomMap() const { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - void refreshAddrToAtomCache() const { - if (!AddrToAtomCache) { - AddrToAtomCache = AddressToAtomMap(); - for (auto *DA : defined_atoms()) - (*AddrToAtomCache)[DA->getAddress()] = const_cast<DefinedAtom *>(DA); - } - } - - // Put the BumpPtrAllocator first so that we don't free any of the atoms in - // it until all of their destructors have been run. - BumpPtrAllocator AtomAllocator; + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; std::string Name; unsigned PointerSize; support::endianness Endianness; + BlockSet Blocks; SectionList Sections; - NamedAtomMap NamedAtoms; - ExternalAtomSet ExternalAtoms; - ExternalAtomSet AbsoluteAtoms; - mutable Optional<AddressToAtomMap> AddrToAtomCache; + ExternalSymbolSet ExternalSymbols; + ExternalSymbolSet AbsoluteSymbols; }; -/// A function for mutating AtomGraphs. -using AtomGraphPassFunction = std::function<Error(AtomGraph &)>; +/// A function for mutating LinkGraphs. +using LinkGraphPassFunction = std::function<Error(LinkGraph &)>; -/// A list of atom graph passes. -using AtomGraphPassList = std::vector<AtomGraphPassFunction>; +/// A list of LinkGraph passes. +using LinkGraphPassList = std::vector<LinkGraphPassFunction>; -/// An atom graph pass configuration, consisting of a list of pre-prune, +/// An LinkGraph pass configuration, consisting of a list of pre-prune, /// post-prune, and post-fixup passes. struct PassConfiguration { /// Pre-prune passes. /// /// These passes are called on the graph after it is built, and before any - /// atoms have been pruned. + /// symbols have been pruned. /// - /// Notable use cases: Marking atoms live or should-discard. - AtomGraphPassList PrePrunePasses; + /// Notable use cases: Marking symbols live or should-discard. + LinkGraphPassList PrePrunePasses; /// Post-prune passes. /// - /// These passes are called on the graph after dead and should-discard atoms - /// have been removed, but before fixups are applied. + /// These passes are called on the graph after dead stripping, but before + /// fixups are applied. /// - /// Notable use cases: Building GOT, stub, and TLV atoms. - AtomGraphPassList PostPrunePasses; + /// Notable use cases: Building GOT, stub, and TLV symbols. + LinkGraphPassList PostPrunePasses; /// Post-fixup passes. /// - /// These passes are called on the graph after atom contents has been copied + /// These passes are called on the graph after block contents has been copied /// to working memory, and fixups applied. /// /// Notable use cases: Testing and validation. - AtomGraphPassList PostFixupPasses; + LinkGraphPassList PostFixupPasses; }; /// A map of symbol names to resolved addresses. using AsyncLookupResult = DenseMap<StringRef, JITEvaluatedSymbol>; -/// A function to call with a resolved symbol map (See AsyncLookupResult) or an -/// error if resolution failed. -using JITLinkAsyncLookupContinuation = - std::function<void(Expected<AsyncLookupResult> LR)>; +/// A function object to call with a resolved symbol map (See AsyncLookupResult) +/// or an error if resolution failed. +class JITLinkAsyncLookupContinuation { +public: + virtual ~JITLinkAsyncLookupContinuation() {} + virtual void run(Expected<AsyncLookupResult> LR) = 0; + +private: + virtual void anchor(); +}; + +/// Create a lookup continuation from a function object. +template <typename Continuation> +std::unique_ptr<JITLinkAsyncLookupContinuation> +createLookupContinuation(Continuation Cont) { -/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) -/// for the given symbols, calling the given continuation with either the result -/// (if the lookup succeeds), or an error (if the lookup fails). -using JITLinkAsyncLookupFunction = - std::function<void(const DenseSet<StringRef> &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation)>; + class Impl final : public JITLinkAsyncLookupContinuation { + public: + Impl(Continuation C) : C(std::move(C)) {} + void run(Expected<AsyncLookupResult> LR) override { C(std::move(LR)); } + + private: + Continuation C; + }; + + return std::make_unique<Impl>(std::move(Cont)); +} /// Holds context for a single jitLink invocation. class JITLinkContext { @@ -881,13 +995,13 @@ public: /// lookup continutation which it must call with a result to continue the /// linking process. virtual void lookup(const DenseSet<StringRef> &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) = 0; + std::unique_ptr<JITLinkAsyncLookupContinuation> LC) = 0; - /// Called by JITLink once all defined atoms in the graph have been assigned - /// their final memory locations in the target process. At this point he - /// atom graph can be, inspected to build a symbol table however the atom + /// Called by JITLink once all defined symbols in the graph have been assigned + /// their final memory locations in the target process. At this point the + /// LinkGraph can be inspected to build a symbol table, however the block /// content will not generally have been copied to the target location yet. - virtual void notifyResolved(AtomGraph &G) = 0; + virtual void notifyResolved(LinkGraph &G) = 0; /// Called by JITLink to notify the context that the object has been /// finalized (i.e. emitted to memory and memory permissions set). If all of @@ -904,20 +1018,20 @@ public: /// Returns the mark-live pass to be used for this link. If no pass is /// returned (the default) then the target-specific linker implementation will - /// choose a conservative default (usually marking all atoms live). + /// choose a conservative default (usually marking all symbols live). /// This function is only called if shouldAddDefaultTargetPasses returns true, /// otherwise the JITContext is responsible for adding a mark-live pass in /// modifyPassConfig. - virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const; + virtual LinkGraphPassFunction getMarkLivePass(const Triple &TT) const; /// Called by JITLink to modify the pass pipeline prior to linking. /// The default version performs no modification. virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config); }; -/// Marks all atoms in a graph live. This can be used as a default, conservative -/// mark-live implementation. -Error markAllAtomsLive(AtomGraph &G); +/// Marks all symbols in a graph live. This can be used as a default, +/// conservative mark-live implementation. +Error markAllSymbolsLive(LinkGraph &G); /// Basic JITLink implementation. /// diff --git a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h index 9d0b37fe4a4d..ac5a593bb77b 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -33,20 +33,19 @@ public: class SegmentRequest { public: SegmentRequest() = default; - SegmentRequest(size_t ContentSize, unsigned ContentAlign, - uint64_t ZeroFillSize, unsigned ZeroFillAlign) - : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize), - ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {} + SegmentRequest(uint64_t Alignment, size_t ContentSize, + uint64_t ZeroFillSize) + : Alignment(Alignment), ContentSize(ContentSize), + ZeroFillSize(ZeroFillSize) { + assert(isPowerOf2_32(Alignment) && "Alignment must be power of 2"); + } + uint64_t getAlignment() const { return Alignment; } size_t getContentSize() const { return ContentSize; } - unsigned getContentAlignment() const { return ContentAlign; } uint64_t getZeroFillSize() const { return ZeroFillSize; } - unsigned getZeroFillAlignment() const { return ZeroFillAlign; } - private: + uint64_t Alignment = 0; size_t ContentSize = 0; uint64_t ZeroFillSize = 0; - unsigned ContentAlign = 0; - unsigned ZeroFillAlign = 0; }; using SegmentsRequestMap = DenseMap<unsigned, SegmentRequest>; diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h new file mode 100644 index 000000000000..d70b545fff86 --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -0,0 +1,60 @@ +//===---- MachO_arm64.h - JIT link functions for MachO/arm64 ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/arm64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_arm64_Edges { + +enum MachOARM64RelocationKind : Edge::Kind { + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + GOTPage21, + GOTPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_arm64_Edges + +/// jit-link the given object buffer, which must be a MachO arm64 object file. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx); + +/// Return the string name of the given MachO arm64 edge kind. +StringRef getMachOARM64RelocationKindName(Edge::Kind R); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h index 1d5b586afc32..00a7feb86e83 100644 --- a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h +++ b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -22,6 +22,7 @@ namespace MachO_x86_64_Edges { enum MachOX86RelocationKind : Edge::Kind { Branch32 = Edge::FirstRelocation, + Pointer32, Pointer64, Pointer64Anon, PCRel32, diff --git a/include/llvm/ExecutionEngine/JITSymbol.h b/include/llvm/ExecutionEngine/JITSymbol.h index b14154c5b5e8..c0f1ca4b9876 100644 --- a/include/llvm/ExecutionEngine/JITSymbol.h +++ b/include/llvm/ExecutionEngine/JITSymbol.h @@ -23,6 +23,7 @@ #include <string> #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" @@ -217,7 +218,7 @@ private: /// Represents a symbol in the JIT. class JITSymbol { public: - using GetAddressFtor = std::function<Expected<JITTargetAddress>()>; + using GetAddressFtor = unique_function<Expected<JITTargetAddress>()>; /// Create a 'null' symbol, used to represent a "symbol not found" /// result from a successful (non-erroneous) lookup. @@ -325,7 +326,7 @@ class JITSymbolResolver { public: using LookupSet = std::set<StringRef>; using LookupResult = std::map<StringRef, JITEvaluatedSymbol>; - using OnResolvedFunction = std::function<void(Expected<LookupResult>)>; + using OnResolvedFunction = unique_function<void(Expected<LookupResult>)>; virtual ~JITSymbolResolver() = default; diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 5f593a27cad6..7946b5b7b209 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -26,6 +26,7 @@ #include "llvm/ExecutionEngine/Orc/LazyReexports.h" #include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constant.h" @@ -91,6 +92,8 @@ public: /// Sets the partition function. void setPartitionFunction(PartitionFunction Partition); + /// Sets the ImplSymbolMap + void setImplMap(ImplSymbolMap *Imp); /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; @@ -128,6 +131,7 @@ private: PerDylibResourcesMap DylibResources; PartitionFunction Partition = compileRequested; SymbolLinkagePromoter PromoteSymbols; + ImplSymbolMap *AliaseeImpls = nullptr; }; /// Compile-on-demand layer. @@ -187,7 +191,7 @@ private: std::unique_ptr<ResourceOwner<ResourceT>> wrapOwnership(ResourcePtrT ResourcePtr) { using RO = ResourceOwnerImpl<ResourceT, ResourcePtrT>; - return llvm::make_unique<RO>(std::move(ResourcePtr)); + return std::make_unique<RO>(std::move(ResourcePtr)); } struct LogicalDylib { @@ -440,7 +444,7 @@ private: return Error::success(); // Create the GlobalValues module. - auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(), + auto GVsM = std::make_unique<Module>((SrcM.getName() + ".globals").str(), SrcM.getContext()); GVsM->setDataLayout(DL); @@ -633,7 +637,7 @@ private: NewName += F->getName(); } - auto M = llvm::make_unique<Module>(NewName, SrcM.getContext()); + auto M = std::make_unique<Module>(NewName, SrcM.getContext()); M->setDataLayout(SrcM.getDataLayout()); ValueToValueMapTy VMap; diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 94a5618233e4..4f22a4c38796 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" @@ -51,8 +52,7 @@ using SymbolMap = DenseMap<SymbolStringPtr, JITEvaluatedSymbol>; /// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. using SymbolFlagsMap = DenseMap<SymbolStringPtr, JITSymbolFlags>; -/// A base class for materialization failures that allows the failing -/// symbols to be obtained for logging. +/// A map from JITDylibs to sets of symbols. using SymbolDependenceMap = DenseMap<JITDylib *, SymbolNameSet>; /// A list of (JITDylib*, bool) pairs. @@ -108,7 +108,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); /// Callback to notify client that symbols have been resolved. -using SymbolsResolvedCallback = std::function<void(Expected<SymbolMap>)>; +using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; /// Callback to register the dependencies for a given query. using RegisterDependenciesFunction = @@ -124,13 +124,13 @@ class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { public: static char ID; - FailedToMaterialize(SymbolNameSet Symbols); + FailedToMaterialize(std::shared_ptr<SymbolDependenceMap> Symbols); std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; - const SymbolNameSet &getSymbols() const { return Symbols; } + const SymbolDependenceMap &getSymbols() const { return *Symbols; } private: - SymbolNameSet Symbols; + std::shared_ptr<SymbolDependenceMap> Symbols; }; /// Used to notify clients when symbols can not be found during a lookup. @@ -205,12 +205,26 @@ public: /// symbols must be ones covered by this MaterializationResponsibility /// instance. Individual calls to this method may resolve a subset of the /// symbols, but all symbols must have been resolved prior to calling emit. - void notifyResolved(const SymbolMap &Symbols); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyResolved(const SymbolMap &Symbols); /// Notifies the target JITDylib (and any pending queries on that JITDylib) /// that all symbols covered by this MaterializationResponsibility instance /// have been emitted. - void notifyEmitted(); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyEmitted(); /// Adds new symbols to the JITDylib and this responsibility instance. /// JITDylib entries start out in the materializing state. @@ -346,7 +360,7 @@ private: /// inline std::unique_ptr<AbsoluteSymbolsMaterializationUnit> absoluteSymbols(SymbolMap Symbols, VModuleKey K = VModuleKey()) { - return llvm::make_unique<AbsoluteSymbolsMaterializationUnit>( + return std::make_unique<AbsoluteSymbolsMaterializationUnit>( std::move(Symbols), std::move(K)); } @@ -390,7 +404,7 @@ private: /// \endcode inline std::unique_ptr<ReExportsMaterializationUnit> symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( nullptr, true, std::move(Aliases), std::move(K)); } @@ -402,7 +416,7 @@ symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { inline std::unique_ptr<ReExportsMaterializationUnit> reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, bool MatchNonExported = false, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( &SourceJD, MatchNonExported, std::move(Aliases), std::move(K)); } @@ -411,32 +425,13 @@ reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); -/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically -/// re-export a subset of the source JITDylib's symbols in the target. -class ReexportsGenerator { -public: - using SymbolPredicate = std::function<bool(SymbolStringPtr)>; - - /// Create a reexports generator. If an Allow predicate is passed, only - /// symbols for which the predicate returns true will be reexported. If no - /// Allow predicate is passed, all symbols will be exported. - ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, - SymbolPredicate Allow = SymbolPredicate()); - - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); - -private: - JITDylib &SourceJD; - bool MatchNonExported = false; - SymbolPredicate Allow; -}; - /// Represents the state that a symbol has reached during materialization. enum class SymbolState : uint8_t { Invalid, /// No symbol should be in this state. NeverSearched, /// Added to the symbol table, never queried. Materializing, /// Queried, materialization begun. Resolved, /// Assigned address, still materializing. + Emitted, /// Emitted to memory, but waiting on transitive dependencies. Ready = 0x3f /// Ready and safe for clients to access. }; @@ -502,8 +497,12 @@ class JITDylib { friend class ExecutionSession; friend class MaterializationResponsibility; public: - using GeneratorFunction = std::function<Expected<SymbolNameSet>( - JITDylib &Parent, const SymbolNameSet &Names)>; + class DefinitionGenerator { + public: + virtual ~DefinitionGenerator(); + virtual Expected<SymbolNameSet> + tryToGenerate(JITDylib &Parent, const SymbolNameSet &Names) = 0; + }; using AsynchronousSymbolQuerySet = std::set<std::shared_ptr<AsynchronousSymbolQuery>>; @@ -519,13 +518,20 @@ public: /// Get a reference to the ExecutionSession for this JITDylib. ExecutionSession &getExecutionSession() const { return ES; } - /// Set a definition generator. If set, whenever a symbol fails to resolve - /// within this JITDylib, lookup and lookupFlags will pass the unresolved - /// symbols set to the definition generator. The generator can optionally - /// add a definition for the unresolved symbols to the dylib. - void setGenerator(GeneratorFunction DefGenerator) { - this->DefGenerator = std::move(DefGenerator); - } + /// Adds a definition generator to this JITDylib and returns a referenece to + /// it. + /// + /// When JITDylibs are searched during lookup, if no existing definition of + /// a symbol is found, then any generators that have been added are run (in + /// the order that they were added) to potentially generate a definition. + template <typename GeneratorT> + GeneratorT &addGenerator(std::unique_ptr<GeneratorT> DefGenerator); + + /// Remove a definition generator from this JITDylib. + /// + /// The given generator must exist in this JITDylib's generators list (i.e. + /// have been added and not yet removed). + void removeGenerator(DefinitionGenerator &G); /// Set the search order to be used when fixing up definitions in JITDylib. /// This will replace the previous search order, and apply to any symbol @@ -633,17 +639,17 @@ private: struct MaterializingInfo { SymbolDependenceMap Dependants; SymbolDependenceMap UnemittedDependencies; - bool IsEmitted = false; void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q); void removeQuery(const AsynchronousSymbolQuery &Q); AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState); - AsynchronousSymbolQueryList takeAllQueries(); + AsynchronousSymbolQueryList takeAllPendingQueries() { + return std::move(PendingQueries); + } bool hasQueriesPending() const { return !PendingQueries.empty(); } const AsynchronousSymbolQueryList &pendingQueries() const { return PendingQueries; } - private: AsynchronousSymbolQueryList PendingQueries; }; @@ -710,9 +716,9 @@ private: SymbolNameSet &Unresolved, bool MatchNonExported, MaterializationUnitList &MUs); - void lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs); bool lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, std::vector<std::unique_ptr<MaterializationUnit>> &MUs, @@ -734,18 +740,20 @@ private: void addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependants); - void resolve(const SymbolMap &Resolved); + Error resolve(const SymbolMap &Resolved); - void emit(const SymbolFlagsMap &Emitted); + Error emit(const SymbolFlagsMap &Emitted); - void notifyFailed(const SymbolNameSet &FailedSymbols); + using FailedSymbolsWorklist = + std::vector<std::pair<JITDylib *, SymbolStringPtr>>; + static void notifyFailed(FailedSymbolsWorklist FailedSymbols); ExecutionSession &ES; std::string JITDylibName; SymbolTable Symbols; UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; - GeneratorFunction DefGenerator; + std::vector<std::unique_ptr<DefinitionGenerator>> DefGenerators; JITDylibSearchList SearchOrder; }; @@ -933,6 +941,14 @@ private: OutstandingMUs; }; +template <typename GeneratorT> +GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { + auto &G = *DefGenerator; + ES.runSessionLocked( + [&]() { DefGenerators.push_back(std::move(DefGenerator)); }); + return G; +} + template <typename Func> auto JITDylib::withSearchOrderDo(Func &&F) -> decltype(F(std::declval<const JITDylibSearchList &>())) { @@ -972,6 +988,27 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) { }); } +/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically +/// re-export a subset of the source JITDylib's symbols in the target. +class ReexportsGenerator : public JITDylib::DefinitionGenerator { +public: + using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + + /// Create a reexports generator. If an Allow predicate is passed, only + /// symbols for which the predicate returns true will be reexported. If no + /// Allow predicate is passed, all symbols will be exported. + ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, + SymbolPredicate Allow = SymbolPredicate()); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + JITDylib &SourceJD; + bool MatchNonExported = false; + SymbolPredicate Allow; +}; + /// Mangles symbol names then uniques them in the context of an /// ExecutionSession. class MangleAndInterner { diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 75865920c741..cf0a428662ef 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -19,6 +19,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Object/Archive.h" #include "llvm/Support/DynamicLibrary.h" #include <algorithm> #include <cstdint> @@ -37,6 +38,8 @@ class Value; namespace orc { +class ObjectLayer; + /// This iterator provides a convenient way to iterate over the elements /// of an llvm.global_ctors/llvm.global_dtors instance. /// @@ -237,7 +240,7 @@ public: /// If an instance of this class is attached to a JITDylib as a fallback /// definition generator, then any symbol found in the given DynamicLibrary that /// passes the 'Allow' predicate will be added to the JITDylib. -class DynamicLibrarySearchGenerator { +class DynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator { public: using SymbolPredicate = std::function<bool(SymbolStringPtr)>; @@ -253,19 +256,20 @@ public: /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> GetForCurrentProcess(char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()) { return Load(nullptr, GlobalPrefix, std::move(Allow)); } - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; private: sys::DynamicLibrary Dylib; @@ -273,6 +277,40 @@ private: char GlobalPrefix; }; +/// A utility class to expose symbols from a static library. +/// +/// If an instance of this class is attached to a JITDylib as a fallback +/// definition generator, then any symbol found in the archive will result in +/// the containing object being added to the JITDylib. +class StaticLibraryDefinitionGenerator : public JITDylib::DefinitionGenerator { +public: + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// is a valid archive, otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName); + + /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. + /// Thhis call will succeed if the buffer contains a valid archive, otherwise + /// it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + StaticLibraryDefinitionGenerator(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> ArchiveBuffer, + Error &Err); + + ObjectLayer &L; + std::unique_ptr<MemoryBuffer> ArchiveBuffer; + object::Archive Archive; + size_t UnrealizedObjects = 0; +}; + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index 1b4c8b6cd95f..b71e5b339711 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -22,6 +22,9 @@ namespace llvm { class Module; namespace orc { +/// A layer that applies a transform to emitted modules. +/// The transform function is responsible for locking the ThreadSafeContext +/// before operating on the module. class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function<Expected<ThreadSafeModule>( diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h index 0aac1916423f..b1e47d77557c 100644 --- a/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -184,8 +184,8 @@ private: class LLJITBuilderState { public: - using ObjectLinkingLayerCreator = - std::function<std::unique_ptr<ObjectLayer>(ExecutionSession &)>; + using ObjectLinkingLayerCreator = std::function<std::unique_ptr<ObjectLayer>( + ExecutionSession &, const Triple &TT)>; using CompileFunctionCreator = std::function<Expected<IRCompileLayer::CompileFunction>( diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 855e31b33549..b31914f12a0d 100644 --- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -16,6 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/OrcV1Deprecation.h" #include <memory> namespace llvm { @@ -62,7 +63,7 @@ std::shared_ptr<LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>> createLambdaResolver(DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(std::move(DylibLookupFtor), + return std::make_unique<LR>(std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } @@ -72,7 +73,7 @@ createLambdaResolver(ORCv1DeprecationAcknowledgement, DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(AcknowledgeORCv1Deprecation, + return std::make_unique<LR>(AcknowledgeORCv1Deprecation, std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 16202d89f861..b67a9feed523 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -49,28 +49,24 @@ private: switch (EmitState) { case NotEmitted: if (auto GV = searchGVs(Name, ExportedSymbolsOnly)) { - // Create a std::string version of Name to capture here - the argument - // (a StringRef) may go away before the lambda is executed. - // FIXME: Use capture-init when we move to C++14. - std::string PName = Name; JITSymbolFlags Flags = JITSymbolFlags::fromGlobalValue(*GV); - auto GetAddress = - [this, ExportedSymbolsOnly, PName, &B]() -> Expected<JITTargetAddress> { - if (this->EmitState == Emitting) - return 0; - else if (this->EmitState == NotEmitted) { - this->EmitState = Emitting; - if (auto Err = this->emitToBaseLayer(B)) - return std::move(Err); - this->EmitState = Emitted; - } - if (auto Sym = B.findSymbolIn(K, PName, ExportedSymbolsOnly)) - return Sym.getAddress(); - else if (auto Err = Sym.takeError()) + auto GetAddress = [this, ExportedSymbolsOnly, Name = Name.str(), + &B]() -> Expected<JITTargetAddress> { + if (this->EmitState == Emitting) + return 0; + else if (this->EmitState == NotEmitted) { + this->EmitState = Emitting; + if (auto Err = this->emitToBaseLayer(B)) return std::move(Err); - else - llvm_unreachable("Successful symbol lookup should return " - "definition address here"); + this->EmitState = Emitted; + } + if (auto Sym = B.findSymbolIn(K, Name, ExportedSymbolsOnly)) + return Sym.getAddress(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + else + llvm_unreachable("Successful symbol lookup should return " + "definition address here"); }; return JITSymbol(std::move(GetAddress), Flags); } else @@ -171,7 +167,7 @@ private: bool ExportedSymbolsOnly) const { assert(!MangledSymbols && "Mangled symbols map already exists?"); - auto Symbols = llvm::make_unique<StringMap<const GlobalValue*>>(); + auto Symbols = std::make_unique<StringMap<const GlobalValue*>>(); Mangler Mang; @@ -209,7 +205,7 @@ public: Error addModule(VModuleKey K, std::unique_ptr<Module> M) { assert(!ModuleMap.count(K) && "VModuleKey K already in use"); ModuleMap[K] = - llvm::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); + std::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); return Error::success(); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/include/llvm/ExecutionEngine/Orc/LazyReexports.h index 9fdd1d15f782..311ed59b1549 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -18,6 +18,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" namespace llvm { @@ -70,7 +71,7 @@ public: template <typename NotifyResolvedImpl> static std::unique_ptr<NotifyResolvedFunction> createNotifyResolvedFunction(NotifyResolvedImpl NotifyResolved) { - return llvm::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( + return std::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( std::move(NotifyResolved)); } @@ -159,7 +160,7 @@ public: IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, - VModuleKey K); + ImplSymbolMap *SrcJDLoc, VModuleKey K); StringRef getName() const override; @@ -174,6 +175,7 @@ private: SymbolAliasMap CallableAliases; std::shared_ptr<LazyCallThroughManager::NotifyResolvedFunction> NotifyResolved; + ImplSymbolMap *AliaseeTable; }; /// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export @@ -182,9 +184,10 @@ private: inline std::unique_ptr<LazyReexportsMaterializationUnit> lazyReexports(LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, - SymbolAliasMap CallableAliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<LazyReexportsMaterializationUnit>( - LCTManager, ISManager, SourceJD, std::move(CallableAliases), + SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc = nullptr, + VModuleKey K = VModuleKey()) { + return std::make_unique<LazyReexportsMaterializationUnit>( + LCTManager, ISManager, SourceJD, std::move(CallableAliases), SrcJDLoc, std::move(K)); } diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h index f9cbbf6ff180..148e260c9569 100644 --- a/include/llvm/ExecutionEngine/Orc/Legacy.h +++ b/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -84,7 +84,7 @@ createSymbolResolver(GetResponsibilitySetFn &&GetResponsibilitySet, typename std::remove_reference<GetResponsibilitySetFn>::type>::type, typename std::remove_cv< typename std::remove_reference<LookupFn>::type>::type>; - return llvm::make_unique<LambdaSymbolResolverImpl>( + return std::make_unique<LambdaSymbolResolverImpl>( std::forward<GetResponsibilitySetFn>(GetResponsibilitySet), std::forward<LookupFn>(Lookup)); } diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index c1e7d27f446e..caf8e707516d 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -73,6 +73,9 @@ public: virtual Error notifyRemovingAllModules() { return Error::success(); } }; + using ReturnObjectBufferFunction = + std::function<void(std::unique_ptr<MemoryBuffer>)>; + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyEmitted functors. ObjectLinkingLayer(ExecutionSession &ES, @@ -81,6 +84,13 @@ public: /// Destruct an ObjectLinkingLayer. ~ObjectLinkingLayer(); + /// Set an object buffer return function. By default object buffers are + /// deleted once the JIT has linked them. If a return function is set then + /// it will be called to transfer ownership of the buffer instead. + void setReturnObjectBuffer(ReturnObjectBufferFunction ReturnObjectBuffer) { + this->ReturnObjectBuffer = std::move(ReturnObjectBuffer); + } + /// Add a pass-config modifier. ObjectLinkingLayer &addPlugin(std::unique_ptr<Plugin> P) { std::lock_guard<std::mutex> Lock(LayerMutex); @@ -138,6 +148,7 @@ private: jitlink::JITLinkMemoryManager &MemMgr; bool OverrideObjectFlags = false; bool AutoClaimObjectSymbols = false; + ReturnObjectBufferFunction ReturnObjectBuffer; DenseMap<VModuleKey, AllocPtr> TrackedAllocs; std::vector<AllocPtr> UntrackedAllocs; std::vector<std::unique_ptr<Plugin>> Plugins; @@ -153,10 +164,16 @@ public: Error notifyRemovingAllModules() override; private: + + struct EHFrameRange { + JITTargetAddress Addr = 0; + size_t Size; + }; + jitlink::EHFrameRegistrar &Registrar; - DenseMap<MaterializationResponsibility *, JITTargetAddress> InProcessLinks; - DenseMap<VModuleKey, JITTargetAddress> TrackedEHFrameAddrs; - std::vector<JITTargetAddress> UntrackedEHFrameAddrs; + DenseMap<MaterializationResponsibility *, EHFrameRange> InProcessLinks; + DenseMap<VModuleKey, EHFrameRange> TrackedEHFrameRanges; + std::vector<EHFrameRange> UntrackedEHFrameRanges; }; } // end namespace orc diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 8b875b7906e1..86e8d5df3ad9 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -493,7 +493,7 @@ public: ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : JITCompileCallbackManager( - llvm::make_unique<RemoteTrampolinePool>(Client), ES, + std::make_unique<RemoteTrampolinePool>(Client), ES, ErrorHandlerAddress) {} }; @@ -553,7 +553,7 @@ public: auto Id = IndirectStubOwnerIds.getNext(); if (auto Err = callB<stubs::CreateIndirectStubsOwner>(Id)) return std::move(Err); - return llvm::make_unique<RemoteIndirectStubsManager>(*this, Id); + return std::make_unique<RemoteIndirectStubsManager>(*this, Id); } Expected<RemoteCompileCallbackManager &> diff --git a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h index 07c7471afc6a..752a0a34e0a1 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h @@ -359,9 +359,9 @@ public: { assert(KeyName != nullptr && "No keyname pointer"); std::lock_guard<std::recursive_mutex> Lock(SerializersMutex); - // FIXME: Move capture Serialize once we have C++14. Serializers[ErrorInfoT::classID()] = - [KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error { + [KeyName, Serialize = std::move(Serialize)]( + ChannelT &C, const ErrorInfoBase &EIB) -> Error { assert(EIB.dynamicClassID() == ErrorInfoT::classID() && "Serializer called for wrong error type"); if (auto Err = serializeSeq(C, *KeyName)) @@ -551,26 +551,26 @@ public: /// RPC channel serialization for std::tuple. static Error serialize(ChannelT &C, const std::tuple<ArgTs...> &V) { - return serializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return serializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } /// RPC channel deserialization for std::tuple. static Error deserialize(ChannelT &C, std::tuple<ArgTs...> &V) { - return deserializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return deserializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } private: // Serialization helper for std::tuple. template <size_t... Is> static Error serializeTupleHelper(ChannelT &C, const std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return serializeSeq(C, std::get<Is>(V)...); } // Serialization helper for std::tuple. template <size_t... Is> static Error deserializeTupleHelper(ChannelT &C, std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return deserializeSeq(C, std::get<Is>(V)...); } }; diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h index 3b11e1b283de..ee9c2cc69c30 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -338,7 +338,9 @@ public: return Err; // Close the response message. - return C.endSendMessage(); + if (auto Err = C.endSendMessage()) + return Err; + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -350,7 +352,9 @@ public: return Err2; if (auto Err2 = serializeSeq(C, std::move(Err))) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -378,8 +382,11 @@ public: C, *ResultOrErr)) return Err; - // Close the response message. - return C.endSendMessage(); + // End the response message. + if (auto Err = C.endSendMessage()) + return Err; + + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -389,7 +396,9 @@ public: return Err; if (auto Err2 = C.startSendMessage(ResponseId, SeqNo)) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -502,7 +511,7 @@ public: static typename WrappedHandlerReturn<RetT>::Type unpackAndRun(HandlerT &Handler, std::tuple<TArgTs...> &Args) { return unpackAndRunHelper(Handler, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -510,7 +519,7 @@ public: static Error unpackAndRunAsync(HandlerT &Handler, ResponderT &Responder, std::tuple<TArgTs...> &Args) { return unpackAndRunAsyncHelper(Handler, Responder, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -540,14 +549,13 @@ public: // Deserialize arguments from the channel. template <typename ChannelT, typename... CArgTs> static Error deserializeArgs(ChannelT &C, std::tuple<CArgTs...> &Args) { - return deserializeArgsHelper(C, Args, - llvm::index_sequence_for<CArgTs...>()); + return deserializeArgsHelper(C, Args, std::index_sequence_for<CArgTs...>()); } private: template <typename ChannelT, typename... CArgTs, size_t... Indexes> static Error deserializeArgsHelper(ChannelT &C, std::tuple<CArgTs...> &Args, - llvm::index_sequence<Indexes...> _) { + std::index_sequence<Indexes...> _) { return SequenceSerialization<ChannelT, ArgTs...>::deserialize( C, std::get<Indexes>(Args)...); } @@ -556,18 +564,16 @@ private: static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunHelper(HandlerT &Handler, ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + std::index_sequence<Indexes...>) { return run(Handler, std::move(std::get<Indexes>(Args))...); } - template <typename HandlerT, typename ResponderT, typename ArgTuple, size_t... Indexes> static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunAsyncHelper(HandlerT &Handler, ResponderT &Responder, - ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + ArgTuple &Args, std::index_sequence<Indexes...>) { return run(Handler, Responder, std::move(std::get<Indexes>(Args))...); } }; @@ -743,11 +749,15 @@ public: // to the user defined handler. Error handleResponse(ChannelT &C) override { Error Result = Error::success(); - if (auto Err = - SerializationTraits<ChannelT, Error, Error>::deserialize(C, Result)) + if (auto Err = SerializationTraits<ChannelT, Error, Error>::deserialize( + C, Result)) { + consumeError(std::move(Result)); return Err; - if (auto Err = C.endReceiveMessage()) + } + if (auto Err = C.endReceiveMessage()) { + consumeError(std::move(Result)); return Err; + } return Handler(std::move(Result)); } @@ -767,7 +777,7 @@ private: // Create a ResponseHandler from a given user handler. template <typename ChannelT, typename FuncRetT, typename HandlerT> std::unique_ptr<ResponseHandler<ChannelT>> createResponseHandler(HandlerT H) { - return llvm::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( + return std::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( std::move(H)); } @@ -1403,14 +1413,12 @@ public: using ErrorReturn = typename RTraits::ErrorReturnType; using ErrorReturnPromise = typename RTraits::ReturnPromiseType; - // FIXME: Stack allocate and move this into the handler once LLVM builds - // with C++14. - auto Promise = std::make_shared<ErrorReturnPromise>(); - auto FutureResult = Promise->get_future(); + ErrorReturnPromise Promise; + auto FutureResult = Promise.get_future(); if (auto Err = this->template appendCallAsync<Func>( - [Promise](ErrorReturn RetOrErr) { - Promise->set_value(std::move(RetOrErr)); + [Promise = std::move(Promise)](ErrorReturn RetOrErr) mutable { + Promise.set_value(std::move(RetOrErr)); return Error::success(); }, Args...)) { @@ -1523,6 +1531,12 @@ public: return std::move(Err); } + if (auto Err = this->C.send()) { + detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( + std::move(Result)); + return std::move(Err); + } + while (!ReceivedResponse) { if (auto Err = this->handleOne()) { detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( @@ -1582,8 +1596,7 @@ public: // outstanding calls count, then poke the condition variable. using ArgType = typename detail::ResponseHandlerArg< typename detail::HandlerTraits<HandlerT>::Type>::ArgType; - // FIXME: Move handler into wrapped handler once we have C++14. - auto WrappedHandler = [this, Handler](ArgType Arg) { + auto WrappedHandler = [this, Handler = std::move(Handler)](ArgType Arg) { auto Err = Handler(std::move(Arg)); std::unique_lock<std::mutex> Lock(M); --NumOutstandingCalls; diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index d9535ce5f21f..c5106cf09ecc 100644 --- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -216,7 +216,7 @@ private: : K(std::move(K)), Parent(Parent), MemMgr(std::move(MemMgr)), - PFC(llvm::make_unique<PreFinalizeContents>( + PFC(std::make_unique<PreFinalizeContents>( std::move(Obj), std::move(Resolver), ProcessAllSections)) { buildInitialSymbolTable(PFC->Obj); @@ -234,7 +234,7 @@ private: JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver, nullptr); - PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); + PFC->RTDyld = std::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections); Finalized = true; @@ -338,7 +338,7 @@ private: std::shared_ptr<SymbolResolver> Resolver, bool ProcessAllSections) { using LOS = ConcreteLinkedObject<MemoryManagerPtrT>; - return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj), + return std::make_unique<LOS>(Parent, std::move(K), std::move(Obj), std::move(MemMgr), std::move(Resolver), ProcessAllSections); } diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h index b87cf697a81e..d7304cfcf931 100644 --- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -137,17 +137,12 @@ protected: RemoteSymbolId Id) : C(C), Id(Id) {} - RemoteSymbolMaterializer(const RemoteSymbolMaterializer &Other) - : C(Other.C), Id(Other.Id) { - // FIXME: This is a horrible, auto_ptr-style, copy-as-move operation. - // It should be removed as soon as LLVM has C++14's generalized - // lambda capture (at which point the materializer can be moved - // into the lambda in remoteToJITSymbol below). - const_cast<RemoteSymbolMaterializer&>(Other).Id = 0; + RemoteSymbolMaterializer(RemoteSymbolMaterializer &&Other) + : C(Other.C), Id(Other.Id) { + Other.Id = 0; } - RemoteSymbolMaterializer& - operator=(const RemoteSymbolMaterializer&) = delete; + RemoteSymbolMaterializer &operator=(RemoteSymbolMaterializer &&) = delete; /// Release the remote symbol. ~RemoteSymbolMaterializer() { @@ -218,9 +213,9 @@ protected: return nullptr; // else... RemoteSymbolMaterializer RSM(*this, RemoteSym.first); - auto Sym = - JITSymbol([RSM]() mutable { return RSM.materialize(); }, - RemoteSym.second); + auto Sym = JITSymbol( + [RSM = std::move(RSM)]() mutable { return RSM.materialize(); }, + RemoteSym.second); return Sym; } else return RemoteSymOrErr.takeError(); @@ -472,7 +467,7 @@ private: } Expected<ObjHandleT> addObject(std::string ObjBuffer) { - auto Buffer = llvm::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); + auto Buffer = std::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); auto Id = HandleIdMgr.getNext(); assert(!BaseLayerHandles.count(Id) && "Id already in use?"); diff --git a/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h new file mode 100644 index 000000000000..cf57b63b6448 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h @@ -0,0 +1,84 @@ +//===-- SpeculateAnalyses.h --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// \file +/// Contains the Analyses and Result Interpretation to select likely functions +/// to Speculatively compile before they are called. [Purely Experimentation] +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H + +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" + +#include <vector> + +namespace llvm { + +namespace orc { + +// Provides common code. +class SpeculateQuery { +protected: + void findCalles(const BasicBlock *, DenseSet<StringRef> &); + bool isStraightLine(const Function &F); + +public: + using ResultTy = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; +}; + +// Direct calls in high frequency basic blocks are extracted. +class BlockFreqQuery : public SpeculateQuery { + size_t numBBToGet(size_t); + +public: + // Find likely next executables based on IR Block Frequency + ResultTy operator()(Function &F); +}; + +// This Query generates a sequence of basic blocks which follows the order of +// execution. +// A handful of BB with higher block frequencies are taken, then path to entry +// and end BB are discovered by traversing up & down the CFG. +class SequenceBBQuery : public SpeculateQuery { + struct WalkDirection { + bool Upward = true, Downward = true; + // the block associated contain a call + bool CallerBlock = false; + }; + +public: + using VisitedBlocksInfoTy = DenseMap<const BasicBlock *, WalkDirection>; + using BlockListTy = SmallVector<const BasicBlock *, 8>; + using BackEdgesInfoTy = + SmallVector<std::pair<const BasicBlock *, const BasicBlock *>, 8>; + using BlockFreqInfoTy = + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8>; + +private: + std::size_t getHottestBlocks(std::size_t TotalBlocks); + BlockListTy rearrangeBB(const Function &, const BlockListTy &); + BlockListTy queryCFG(Function &, const BlockListTy &); + void traverseToEntryBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + void traverseToExitBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + +public: + ResultTy operator()(Function &F); +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H diff --git a/include/llvm/ExecutionEngine/Orc/Speculation.h b/include/llvm/ExecutionEngine/Orc/Speculation.h new file mode 100644 index 000000000000..766a6b070f12 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -0,0 +1,207 @@ +//===-- Speculation.h - Speculative Compilation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition to support speculative compilation when laziness is +// enabled. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATION_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/Debug.h" + +#include <mutex> +#include <type_traits> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { + +class Speculator; + +// Track the Impls (JITDylib,Symbols) of Symbols while lazy call through +// trampolines are created. Operations are guarded by locks tp ensure that Imap +// stays in consistent state after read/write + +class ImplSymbolMap { + friend class Speculator; + +public: + using AliaseeDetails = std::pair<SymbolStringPtr, JITDylib *>; + using Alias = SymbolStringPtr; + using ImapTy = DenseMap<Alias, AliaseeDetails>; + void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD); + +private: + // FIX ME: find a right way to distinguish the pre-compile Symbols, and update + // the callsite + Optional<AliaseeDetails> getImplFor(const SymbolStringPtr &StubSymbol) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto Position = Maps.find(StubSymbol); + if (Position != Maps.end()) + return Position->getSecond(); + else + return None; + } + + std::mutex ConcurrentAccess; + ImapTy Maps; +}; + +// Defines Speculator Concept, +class Speculator { +public: + using TargetFAddr = JITTargetAddress; + using FunctionCandidatesMap = DenseMap<SymbolStringPtr, SymbolNameSet>; + using StubAddrLikelies = DenseMap<TargetFAddr, SymbolNameSet>; + +private: + void registerSymbolsWithAddr(TargetFAddr ImplAddr, + SymbolNameSet likelySymbols) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + GlobalSpecMap.insert({ImplAddr, std::move(likelySymbols)}); + } + + void launchCompile(JITTargetAddress FAddr) { + SymbolNameSet CandidateSet; + // Copy CandidateSet is necessary, to avoid unsynchronized access to + // the datastructure. + { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto It = GlobalSpecMap.find(FAddr); + if (It == GlobalSpecMap.end()) + return; + CandidateSet = It->getSecond(); + } + + SymbolDependenceMap SpeculativeLookUpImpls; + + for (auto &Callee : CandidateSet) { + auto ImplSymbol = AliaseeImplTable.getImplFor(Callee); + // try to distinguish already compiled & library symbols + if (!ImplSymbol.hasValue()) + continue; + const auto &ImplSymbolName = ImplSymbol.getPointer()->first; + JITDylib *ImplJD = ImplSymbol.getPointer()->second; + auto &SymbolsInJD = SpeculativeLookUpImpls[ImplJD]; + SymbolsInJD.insert(ImplSymbolName); + } + + DEBUG_WITH_TYPE("orc", for (auto &I + : SpeculativeLookUpImpls) { + llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; + for (auto &N : I.second) + llvm::dbgs() << "\n Likely Symbol : " << N; + }); + + // for a given symbol, there may be no symbol qualified for speculatively + // compile try to fix this before jumping to this code if possible. + for (auto &LookupPair : SpeculativeLookUpImpls) + ES.lookup(JITDylibSearchList({{LookupPair.first, true}}), + LookupPair.second, SymbolState::Ready, + [this](Expected<SymbolMap> Result) { + if (auto Err = Result.takeError()) + ES.reportError(std::move(Err)); + }, + NoDependenciesToRegister); + } + +public: + Speculator(ImplSymbolMap &Impl, ExecutionSession &ref) + : AliaseeImplTable(Impl), ES(ref), GlobalSpecMap(0) {} + Speculator(const Speculator &) = delete; + Speculator(Speculator &&) = delete; + Speculator &operator=(const Speculator &) = delete; + Speculator &operator=(Speculator &&) = delete; + + /// Define symbols for this Speculator object (__orc_speculator) and the + /// speculation runtime entry point symbol (__orc_speculate_for) in the + /// given JITDylib. + Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle); + + // Speculatively compile likely functions for the given Stub Address. + // destination of __orc_speculate_for jump + void speculateFor(TargetFAddr StubAddr) { launchCompile(StubAddr); } + + // FIXME : Register with Stub Address, after JITLink Fix. + void registerSymbols(FunctionCandidatesMap Candidates, JITDylib *JD) { + for (auto &SymPair : Candidates) { + auto Target = SymPair.first; + auto Likely = SymPair.second; + + auto OnReadyFixUp = [Likely, Target, + this](Expected<SymbolMap> ReadySymbol) { + if (ReadySymbol) { + auto RAddr = (*ReadySymbol)[Target].getAddress(); + registerSymbolsWithAddr(RAddr, std::move(Likely)); + } else + this->getES().reportError(ReadySymbol.takeError()); + }; + // Include non-exported symbols also. + ES.lookup(JITDylibSearchList({{JD, true}}), SymbolNameSet({Target}), + SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); + } + } + + ExecutionSession &getES() { return ES; } + +private: + static void speculateForEntryPoint(Speculator *Ptr, uint64_t StubId); + std::mutex ConcurrentAccess; + ImplSymbolMap &AliaseeImplTable; + ExecutionSession &ES; + StubAddrLikelies GlobalSpecMap; +}; + +class IRSpeculationLayer : public IRLayer { +public: + using IRlikiesStrRef = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; + using ResultEval = std::function<IRlikiesStrRef(Function &)>; + using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>; + + IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, + Speculator &Spec, MangleAndInterner &Mangle, + ResultEval Interpreter) + : IRLayer(ES), NextLayer(BaseLayer), S(Spec), Mangle(Mangle), + QueryAnalysis(Interpreter) {} + + void emit(MaterializationResponsibility R, ThreadSafeModule TSM); + +private: + TargetAndLikelies + internToJITSymbols(DenseMap<StringRef, DenseSet<StringRef>> IRNames) { + assert(!IRNames.empty() && "No IRNames received to Intern?"); + TargetAndLikelies InternedNames; + DenseSet<SymbolStringPtr> TargetJITNames; + for (auto &NamePair : IRNames) { + for (auto &TargetNames : NamePair.second) + TargetJITNames.insert(Mangle(TargetNames)); + + InternedNames[Mangle(NamePair.first)] = std::move(TargetJITNames); + } + return InternedNames; + } + + IRCompileLayer &NextLayer; + Speculator &S; + MangleAndInterner &Mangle; + ResultEval QueryAnalysis; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATION_H diff --git a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h index 5787500387c4..2347faed37a2 100644 --- a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h +++ b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -38,17 +38,12 @@ private: public: // RAII based lock for ThreadSafeContext. class LLVM_NODISCARD Lock { - private: - using UnderlyingLock = std::lock_guard<std::recursive_mutex>; - public: - Lock(std::shared_ptr<State> S) - : S(std::move(S)), - L(llvm::make_unique<UnderlyingLock>(this->S->Mutex)) {} + Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {} private: std::shared_ptr<State> S; - std::unique_ptr<UnderlyingLock> L; + std::unique_lock<std::recursive_mutex> L; }; /// Construct a null context. @@ -69,7 +64,7 @@ public: /// instance, or null if the instance was default constructed. const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; } - Lock getLock() { + Lock getLock() const { assert(S && "Can not lock an empty ThreadSafeContext"); return Lock(S); } @@ -95,7 +90,7 @@ public: // We also need to lock the context to make sure the module tear-down // does not overlap any other work on the context. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } M = std::move(Other.M); @@ -117,23 +112,14 @@ public: ~ThreadSafeModule() { // We need to lock the context while we destruct the module. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } } - /// Get the module wrapped by this ThreadSafeModule. - Module *getModule() { return M.get(); } - - /// Get the module wrapped by this ThreadSafeModule. - const Module *getModule() const { return M.get(); } - - /// Take out a lock on the ThreadSafeContext for this module. - ThreadSafeContext::Lock getContextLock() { return TSCtx.getLock(); } - /// Boolean conversion: This ThreadSafeModule will evaluate to true if it /// wraps a non-null module. - explicit operator bool() { + explicit operator bool() const { if (M) { assert(TSCtx.getContext() && "Non-null module must have non-null context"); @@ -142,6 +128,33 @@ public: return false; } + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) -> decltype(F(std::declval<Module &>())) { + assert(M && "Can not call on null module"); + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) const + -> decltype(F(std::declval<const Module &>())) { + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Get a raw pointer to the contained module without locking the context. + Module *getModuleUnlocked() { return M.get(); } + + /// Get a raw pointer to the contained module without locking the context. + const Module *getModuleUnlocked() const { return M.get(); } + + /// Returns the context for this ThreadSafeModule. + ThreadSafeContext getContext() const { return TSCtx; } + private: std::unique_ptr<Module> M; ThreadSafeContext TSCtx; diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h index b2b4eba47074..ce7024a7f19b 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -13,6 +13,7 @@ #ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H #define LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/DIContext.h" @@ -271,10 +272,10 @@ private: std::unique_ptr<MemoryBuffer> UnderlyingBuffer, RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, bool ProcessAllSections, - std::function<Error(std::unique_ptr<LoadedObjectInfo>, - std::map<StringRef, JITEvaluatedSymbol>)> + unique_function<Error(std::unique_ptr<LoadedObjectInfo>, + std::map<StringRef, JITEvaluatedSymbol>)> OnLoaded, - std::function<void(Error)> OnEmitted); + unique_function<void(Error)> OnEmitted); // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public // interface. @@ -291,14 +292,14 @@ private: // but ORC's RTDyldObjectLinkingLayer2. Internally it constructs a RuntimeDyld // instance and uses continuation passing to perform the fix-up and finalize // steps asynchronously. -void jitLinkForORC(object::ObjectFile &Obj, - std::unique_ptr<MemoryBuffer> UnderlyingBuffer, - RuntimeDyld::MemoryManager &MemMgr, - JITSymbolResolver &Resolver, bool ProcessAllSections, - std::function<Error(std::unique_ptr<LoadedObjectInfo>, - std::map<StringRef, JITEvaluatedSymbol>)> - OnLoaded, - std::function<void(Error)> OnEmitted); +void jitLinkForORC( + object::ObjectFile &Obj, std::unique_ptr<MemoryBuffer> UnderlyingBuffer, + RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, + bool ProcessAllSections, + unique_function<Error(std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, + std::map<StringRef, JITEvaluatedSymbol>)> + OnLoaded, + unique_function<void(Error)> OnEmitted); } // end namespace llvm |