diff options
Diffstat (limited to 'llvm/include/llvm/ExecutionEngine')
38 files changed, 1568 insertions, 681 deletions
diff --git a/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h b/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h index 4fb6dad96387..2562da7cf60b 100644 --- a/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h +++ b/llvm/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -158,6 +158,8 @@ protected: /// getMangledName - Get mangled name. std::string getMangledName(const GlobalValue *GV); + std::string ErrMsg; + public: /// lock - This lock protects the ExecutionEngine and MCJIT classes. It must /// be held while changing the internal state of any of those classes. @@ -275,8 +277,20 @@ public: /// object have been relocated using mapSectionAddress. When this method is /// called the MCJIT execution engine will reapply relocations for a loaded /// object. This method has no effect for the interpeter. + /// + /// Returns true on success, false on failure. Error messages can be retrieved + /// by calling getError(); virtual void finalizeObject() {} + /// Returns true if an error has been recorded. + bool hasError() const { return !ErrMsg.empty(); } + + /// Clear the error message. + void clearErrorMessage() { ErrMsg.clear(); } + + /// Returns the most recent error message. + const std::string &getErrorMessage() const { return ErrMsg; } + /// runStaticConstructorsDestructors - This method is used to execute all of /// the static constructors or destructors for a program. /// @@ -499,7 +513,7 @@ protected: void emitGlobals(); - void EmitGlobalVariable(const GlobalVariable *GV); + void emitGlobalVariable(const GlobalVariable *GV); GenericValue getConstantValue(const Constant *C); void LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr, diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h new file mode 100644 index 000000000000..9f6ea5271f4b --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF.h @@ -0,0 +1,31 @@ +//===------- ELF.h - Generic JIT link function for ELF ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic jit-link functions for ELF. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_H + +#include "llvm/ExecutionEngine/JITLink/ELF.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// jit-link the given ObjBuffer, which must be a ELF object file. +/// +/// Uses conservative defaults for GOT and stub handling based on the target +/// platform. +void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_H diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h new file mode 100644 index 000000000000..7860088f3569 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h @@ -0,0 +1,52 @@ +//===--- ELF_x86_64.h - JIT link functions for ELF/x86-64 ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for ELF/x86-64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace ELF_x86_64_Edges { +enum ELFX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Branch32ToStub, + Pointer32, + Pointer64, + Pointer64Anon, + PCRel32, + PCRel32Minus1, + PCRel32Minus2, + PCRel32Minus4, + PCRel32Anon, + PCRel32Minus1Anon, + PCRel32Minus2Anon, + PCRel32Minus4Anon, + PCRel32GOTLoad, + PCRel32GOT, + PCRel32TLV, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // end namespace ELF_x86_64_Edges + +/// jit-link the given object buffer, which must be a ELF x86-64 object file. +void jitLink_ELF_x86_64(std::unique_ptr<JITLinkContext> Ctx); +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index fa04653fa7bd..76f9dea4160f 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -17,6 +17,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/Support/Allocator.h" @@ -247,8 +248,8 @@ public: bool edges_empty() const { return Edges.empty(); } /// Remove the edge pointed to by the given iterator. - /// Invalidates all iterators that point to or past the given one. - void removeEdge(const_edge_iterator I) { Edges.erase(I); } + /// Returns an iterator to the new next element. + edge_iterator removeEdge(edge_iterator I) { return Edges.erase(I); } private: static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1; @@ -351,7 +352,8 @@ private: JITTargetAddress Size, bool IsCallable, bool IsLive) { assert(SymStorage && "Storage cannot be null"); - assert(Offset < Base.getSize() && "Symbol offset is outside block"); + assert((Offset + Size) <= Base.getSize() && + "Symbol extends past end of block"); auto *Sym = reinterpret_cast<Symbol *>(SymStorage); new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, Scope::Local, IsLive, IsCallable); @@ -363,7 +365,8 @@ private: JITTargetAddress Size, Linkage L, Scope S, bool IsLive, bool IsCallable) { assert(SymStorage && "Storage cannot be null"); - assert(Offset < Base.getSize() && "Symbol offset is outside block"); + assert((Offset + Size) <= Base.getSize() && + "Symbol extends past end of block"); assert(!Name.empty() && "Name cannot be empty"); auto *Sym = reinterpret_cast<Symbol *>(SymStorage); new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); @@ -487,6 +490,8 @@ public: /// Set the visibility for this Symbol. void setScope(Scope S) { + assert((!Name.empty() || S == Scope::Local) && + "Can not set anonymous symbol to non-local scope"); assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && "Invalid visibility for symbol type"); this->S = static_cast<uint8_t>(S); @@ -990,6 +995,11 @@ public: /// Remove a block. void removeBlock(Block &B) { + assert(llvm::none_of(B.getSection().symbols(), + [&](const Symbol *Sym) { + return &Sym->getBlock() == &B; + }) && + "Block still has symbols attached"); B.getSection().removeBlock(B); destroyBlock(B); } @@ -1168,7 +1178,7 @@ struct PassConfiguration { /// Pre-prune passes. /// /// These passes are called on the graph after it is built, and before any - /// symbols have been pruned. + /// symbols have been pruned. Graph nodes still have their original vmaddrs. /// /// Notable use cases: Marking symbols live or should-discard. LinkGraphPassList PrePrunePasses; @@ -1176,15 +1186,26 @@ struct PassConfiguration { /// Post-prune passes. /// /// These passes are called on the graph after dead stripping, but before - /// fixups are applied. + /// memory is allocated or nodes assigned their final addresses. /// /// Notable use cases: Building GOT, stub, and TLV symbols. LinkGraphPassList PostPrunePasses; + /// Pre-fixup passes. + /// + /// These passes are called on the graph after memory has been allocated, + /// content copied into working memory, and nodes have been assigned their + /// final addresses. + /// + /// Notable use cases: Late link-time optimizations like GOT and stub + /// elimination. + LinkGraphPassList PostAllocationPasses; + /// Post-fixup passes. /// /// These passes are called on the graph after block contents has been copied - /// to working memory, and fixups applied. + /// to working memory, and fixups applied. Graph nodes have been updated to + /// their final target vmaddrs. /// /// Notable use cases: Testing and validation. LinkGraphPassList PostFixupPasses; diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h index ac5a593bb77b..0c8514a60a50 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -17,7 +17,10 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/Support/Error.h" #include "llvm/Support/Memory.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" + #include <cstdint> +#include <future> namespace llvm { namespace jitlink { @@ -74,6 +77,15 @@ public: /// working memory. virtual void finalizeAsync(FinalizeContinuation OnFinalize) = 0; + /// Calls finalizeAsync and waits for completion. + Error finalize() { + std::promise<MSVCPError> FinalizeResultP; + auto FinalizeResultF = FinalizeResultP.get_future(); + finalizeAsync( + [&](Error Err) { FinalizeResultP.set_value(std::move(Err)); }); + return FinalizeResultF.get(); + } + /// Should deallocate target memory. virtual Error deallocate() = 0; }; diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h index 00a7feb86e83..27fcdf4fa990 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -22,6 +22,7 @@ namespace MachO_x86_64_Edges { enum MachOX86RelocationKind : Edge::Kind { Branch32 = Edge::FirstRelocation, + Branch32ToStub, Pointer32, Pointer64, Pointer64Anon, diff --git a/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/llvm/include/llvm/ExecutionEngine/JITSymbol.h index 7a2a6cfa5203..6f0030a18f47 100644 --- a/llvm/include/llvm/ExecutionEngine/JITSymbol.h +++ b/llvm/include/llvm/ExecutionEngine/JITSymbol.h @@ -30,6 +30,7 @@ namespace llvm { class GlobalValue; +class GlobalValueSummary; namespace object { @@ -58,10 +59,9 @@ template <typename T> T jitTargetAddressToPointer(JITTargetAddress Addr) { /// Casts the given address to a callable function pointer. This operation /// will perform pointer signing for platforms that require it (e.g. arm64e). template <typename T> T jitTargetAddressToFunction(JITTargetAddress Addr) { - static_assert( - std::is_pointer<T>::value && - std::is_function<typename std::remove_pointer<T>::type>::value, - "T must be a function pointer type"); + static_assert(std::is_pointer<T>::value && + std::is_function<std::remove_pointer_t<T>>::value, + "T must be a function pointer type"); return jitTargetAddressToPointer<T>(Addr); } @@ -84,7 +84,9 @@ public: Absolute = 1U << 3, Exported = 1U << 4, Callable = 1U << 5, - LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable) + MaterializationSideEffectsOnly = 1U << 6, + LLVM_MARK_AS_BITMASK_ENUM( // LargestValue = + MaterializationSideEffectsOnly) }; /// Default-construct a JITSymbolFlags instance. @@ -146,6 +148,21 @@ public: /// Returns true if the given symbol is known to be callable. bool isCallable() const { return (Flags & Callable) == Callable; } + /// Returns true if this symbol is a materialization-side-effects-only + /// symbol. Such symbols do not have a real address. They exist to trigger + /// and support synchronization of materialization side effects, e.g. for + /// collecting initialization information. These symbols will vanish from + /// the symbol table immediately upon reaching the ready state, and will + /// appear to queries as if they were never defined (except that query + /// callback execution will be delayed until they reach the ready state). + /// MaterializationSideEffectOnly symbols should only be queried using the + /// SymbolLookupFlags::WeaklyReferencedSymbol flag (see + /// llvm/include/llvm/ExecutionEngine/Orc/Core.h). + bool hasMaterializationSideEffectsOnly() const { + return (Flags & MaterializationSideEffectsOnly) == + MaterializationSideEffectsOnly; + } + /// Get the underlying flags value as an integer. UnderlyingType getRawFlagsValue() const { return static_cast<UnderlyingType>(Flags); @@ -161,6 +178,10 @@ public: /// value. static JITSymbolFlags fromGlobalValue(const GlobalValue &GV); + /// Construct a JITSymbolFlags value based on the flags of the given global + /// value summary. + static JITSymbolFlags fromSummary(GlobalValueSummary *S); + /// Construct a JITSymbolFlags value based on the flags of the given libobject /// symbol. static Expected<JITSymbolFlags> @@ -216,6 +237,13 @@ public: JITEvaluatedSymbol(JITTargetAddress Address, JITSymbolFlags Flags) : Address(Address), Flags(Flags) {} + /// Create a symbol from the given pointer with the given flags. + template <typename T> + static JITEvaluatedSymbol + fromPointer(T *P, JITSymbolFlags Flags = JITSymbolFlags::Exported) { + return JITEvaluatedSymbol(pointerToJITTargetAddress(P), Flags); + } + /// An evaluated symbol converts to 'true' if its address is non-zero. explicit operator bool() const { return Address != 0; } diff --git a/llvm/include/llvm/ExecutionEngine/ObjectCache.h b/llvm/include/llvm/ExecutionEngine/ObjectCache.h index 47e94f18a1c7..1c72ca39f7c1 100644 --- a/llvm/include/llvm/ExecutionEngine/ObjectCache.h +++ b/llvm/include/llvm/ExecutionEngine/ObjectCache.h @@ -9,11 +9,12 @@ #ifndef LLVM_EXECUTIONENGINE_OBJECTCACHE_H #define LLVM_EXECUTIONENGINE_OBJECTCACHE_H -#include "llvm/Support/MemoryBuffer.h" #include <memory> namespace llvm { +class MemoryBuffer; +class MemoryBufferRef; class Module; /// This is the base ObjectCache type which can be provided to an diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 7946b5b7b209..9ecc0464dec1 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -18,7 +18,6 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Twine.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/LambdaResolver.h" @@ -94,6 +93,7 @@ public: /// Sets the ImplSymbolMap void setImplMap(ImplSymbolMap *Imp); + /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; @@ -338,12 +338,13 @@ public: for (auto &KV : LogicalDylibs) { if (auto Sym = KV.second.StubsMgr->findStub(Name, ExportedSymbolsOnly)) return Sym; - if (auto Sym = findSymbolIn(KV.first, Name, ExportedSymbolsOnly)) + if (auto Sym = + findSymbolIn(KV.first, std::string(Name), ExportedSymbolsOnly)) return Sym; else if (auto Err = Sym.takeError()) return std::move(Err); } - return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); + return BaseLayer.findSymbol(std::string(Name), ExportedSymbolsOnly); } /// Get the address of a symbol provided by this layer, or some layer @@ -392,50 +393,49 @@ private: // Create stub functions. const DataLayout &DL = SrcM.getDataLayout(); - { - typename IndirectStubsMgrT::StubInitsMap StubInits; - for (auto &F : SrcM) { - // Skip declarations. - if (F.isDeclaration()) - continue; - // Skip weak functions for which we already have definitions. - auto MangledName = mangle(F.getName(), DL); - if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) { - if (auto Sym = LD.findSymbol(BaseLayer, MangledName, false)) - continue; - else if (auto Err = Sym.takeError()) - return std::move(Err); - } + typename IndirectStubsMgrT::StubInitsMap StubInits; + for (auto &F : SrcM) { + // Skip declarations. + if (F.isDeclaration()) + continue; - // Record all functions defined by this module. - if (CloneStubsIntoPartitions) - LD.getStubsToClone(LMId).insert(&F); - - // Create a callback, associate it with the stub for the function, - // and set the compile action to compile the partition containing the - // function. - auto CompileAction = [this, &LD, LMId, &F]() -> JITTargetAddress { - if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) - return *FnImplAddrOrErr; - else { - // FIXME: Report error, return to 'abort' or something similar. - consumeError(FnImplAddrOrErr.takeError()); - return 0; - } - }; - if (auto CCAddr = - CompileCallbackMgr.getCompileCallback(std::move(CompileAction))) - StubInits[MangledName] = - std::make_pair(*CCAddr, JITSymbolFlags::fromGlobalValue(F)); - else - return CCAddr.takeError(); + // Skip weak functions for which we already have definitions. + auto MangledName = mangle(F.getName(), DL); + if (F.hasWeakLinkage() || F.hasLinkOnceLinkage()) { + if (auto Sym = LD.findSymbol(BaseLayer, MangledName, false)) + continue; + else if (auto Err = Sym.takeError()) + return Err; } - if (auto Err = LD.StubsMgr->createStubs(StubInits)) - return Err; + // Record all functions defined by this module. + if (CloneStubsIntoPartitions) + LD.getStubsToClone(LMId).insert(&F); + + // Create a callback, associate it with the stub for the function, + // and set the compile action to compile the partition containing the + // function. + auto CompileAction = [this, &LD, LMId, &F]() -> JITTargetAddress { + if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) + return *FnImplAddrOrErr; + else { + // FIXME: Report error, return to 'abort' or something similar. + consumeError(FnImplAddrOrErr.takeError()); + return 0; + } + }; + if (auto CCAddr = + CompileCallbackMgr.getCompileCallback(std::move(CompileAction))) + StubInits[MangledName] = + std::make_pair(*CCAddr, JITSymbolFlags::fromGlobalValue(F)); + else + return CCAddr.takeError(); } + if (auto Err = LD.StubsMgr->createStubs(StubInits)) + return Err; + // If this module doesn't contain any globals, aliases, or module flags then // we can bail out early and avoid the overhead of creating and managing an // empty globals module. @@ -511,11 +511,11 @@ private: } // Build a resolver for the globals module and add it to the base layer. - auto LegacyLookup = [this, &LD](const std::string &Name) -> JITSymbol { + auto LegacyLookup = [this, &LD](StringRef Name) -> JITSymbol { if (auto Sym = LD.StubsMgr->findStub(Name, false)) return Sym; - if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) + if (auto Sym = LD.findSymbol(BaseLayer, std::string(Name), false)) return Sym; else if (auto Err = Sym.takeError()) return std::move(Err); @@ -631,7 +631,7 @@ private: Module &SrcM = LD.getSourceModule(LMId); // Create the module. - std::string NewName = SrcM.getName(); + std::string NewName(SrcM.getName()); for (auto *F : Part) { NewName += "."; NewName += F->getName(); @@ -688,8 +688,8 @@ private: auto K = ES.allocateVModule(); - auto LegacyLookup = [this, &LD](const std::string &Name) -> JITSymbol { - return LD.findSymbol(BaseLayer, Name, false); + auto LegacyLookup = [this, &LD](StringRef Name) -> JITSymbol { + return LD.findSymbol(BaseLayer, std::string(Name), false); }; // Create memory manager and symbol resolver. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h index eb6d84e8cbb4..8376d163d57a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -13,7 +13,9 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H #define LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" #include <memory> namespace llvm { @@ -28,24 +30,31 @@ namespace orc { class JITTargetMachineBuilder; +IRSymbolMapper::ManglingOptions +irManglingOptionsFromTargetOptions(const TargetOptions &Opts); + /// Simple compile functor: Takes a single IR module and returns an ObjectFile. /// This compiler supports a single compilation thread and LLVMContext only. /// For multithreaded compilation, use ConcurrentIRCompiler below. -class SimpleCompiler { +class SimpleCompiler : public IRCompileLayer::IRCompiler { public: using CompileResult = std::unique_ptr<MemoryBuffer>; /// Construct a simple compile functor with the given target. SimpleCompiler(TargetMachine &TM, ObjectCache *ObjCache = nullptr) - : TM(TM), ObjCache(ObjCache) {} + : IRCompiler(irManglingOptionsFromTargetOptions(TM.Options)), TM(TM), + ObjCache(ObjCache) {} /// Set an ObjectCache to query before compiling. void setObjectCache(ObjectCache *NewCache) { ObjCache = NewCache; } /// Compile a Module to an ObjectFile. - CompileResult operator()(Module &M); + Expected<CompileResult> operator()(Module &M) override; private: + IRSymbolMapper::ManglingOptions + manglingOptionsForTargetMachine(const TargetMachine &TM); + CompileResult tryToLoadFromObjectCache(const Module &M); void notifyObjectCompiled(const Module &M, const MemoryBuffer &ObjBuffer); @@ -73,14 +82,14 @@ private: /// /// This class creates a new TargetMachine and SimpleCompiler instance for each /// compile. -class ConcurrentIRCompiler { +class ConcurrentIRCompiler : public IRCompileLayer::IRCompiler { public: ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, ObjectCache *ObjCache = nullptr); void setObjectCache(ObjectCache *ObjCache) { this->ObjCache = ObjCache; } - std::unique_ptr<MemoryBuffer> operator()(Module &M); + Expected<std::unique_ptr<MemoryBuffer>> operator()(Module &M) override; private: JITTargetMachineBuilder JTMB; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index d0a9ca5c0580..a117acefd2d3 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,18 +14,16 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" -#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include <memory> #include <vector> -#define DEBUG_TYPE "orc" - namespace llvm { namespace orc { @@ -202,10 +200,10 @@ public: /// If Body returns true then the element just passed in is removed from the /// set. If Body returns false then the element is retained. template <typename BodyFn> - auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if< + auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t< std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), std::declval<SymbolLookupFlags>())), - bool>::value>::type { + bool>::value> { UnderlyingVector::size_type I = 0; while (I != Symbols.size()) { const auto &Name = Symbols[I].first; @@ -224,11 +222,11 @@ public: /// returns true then the element just passed in is removed from the set. If /// Body returns false then the element is retained. template <typename BodyFn> - auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if< + auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t< std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), std::declval<SymbolLookupFlags>())), Expected<bool>>::value, - Error>::type { + Error> { UnderlyingVector::size_type I = 0; while (I != Symbols.size()) { const auto &Name = Symbols[I].first; @@ -309,66 +307,6 @@ struct SymbolAliasMapEntry { /// A map of Symbols to (Symbol, Flags) pairs. using SymbolAliasMap = DenseMap<SymbolStringPtr, SymbolAliasMapEntry>; -/// Render a SymbolStringPtr. -raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym); - -/// Render a SymbolNameSet. -raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols); - -/// Render a SymbolNameVector. -raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols); - -/// Render a SymbolFlagsMap entry. -raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV); - -/// Render a SymbolMap entry. -raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV); - -/// Render a SymbolFlagsMap. -raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags); - -/// Render a SymbolMap. -raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols); - -/// Render a SymbolDependenceMap entry. -raw_ostream &operator<<(raw_ostream &OS, - const SymbolDependenceMap::value_type &KV); - -/// Render a SymbolDependendeMap. -raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps); - -/// Render a MaterializationUnit. -raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU); - -//// Render a JITDylibLookupFlags instance. -raw_ostream &operator<<(raw_ostream &OS, - const JITDylibLookupFlags &JDLookupFlags); - -/// Rendar a SymbolLookupFlags instance. -raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags); - -/// Render a JITDylibLookupFlags instance. -raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); - -/// Render a SymbolLookupSet entry. -raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV); - -/// Render a SymbolLookupSet. -raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet); - -/// Render a JITDylibSearchOrder. -raw_ostream &operator<<(raw_ostream &OS, - const JITDylibSearchOrder &SearchOrder); - -/// Render a SymbolAliasMap. -raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); - -/// Render a SymbolState. -raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); - -/// Render a LookupKind. -raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); - /// Callback to notify client that symbols have been resolved. using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; @@ -424,6 +362,44 @@ private: SymbolNameSet Symbols; }; +/// Errors of this type should be returned if a module fails to include +/// definitions that are claimed by the module's associated +/// MaterializationResponsibility. If this error is returned it is indicative of +/// a broken transformation / compiler / object cache. +class MissingSymbolDefinitions : public ErrorInfo<MissingSymbolDefinitions> { +public: + static char ID; + + MissingSymbolDefinitions(std::string ModuleName, SymbolNameVector Symbols) + : ModuleName(std::move(ModuleName)), Symbols(std::move(Symbols)) {} + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getModuleName() const { return ModuleName; } + const SymbolNameVector &getSymbols() const { return Symbols; } +private: + std::string ModuleName; + SymbolNameVector Symbols; +}; + +/// Errors of this type should be returned if a module contains definitions for +/// symbols that are not claimed by the module's associated +/// MaterializationResponsibility. If this error is returned it is indicative of +/// a broken transformation / compiler / object cache. +class UnexpectedSymbolDefinitions : public ErrorInfo<UnexpectedSymbolDefinitions> { +public: + static char ID; + + UnexpectedSymbolDefinitions(std::string ModuleName, SymbolNameVector Symbols) + : ModuleName(std::move(ModuleName)), Symbols(std::move(Symbols)) {} + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getModuleName() const { return ModuleName; } + const SymbolNameVector &getSymbols() const { return Symbols; } +private: + std::string ModuleName; + SymbolNameVector Symbols; +}; + /// Tracks responsibility for materialization, and mediates interactions between /// MaterializationUnits and JDs. /// @@ -445,7 +421,7 @@ public: /// Returns the target JITDylib that these symbols are being materialized /// into. - JITDylib &getTargetJITDylib() const { return JD; } + JITDylib &getTargetJITDylib() const { return *JD; } /// Returns the VModuleKey for this instance. VModuleKey getVModuleKey() const { return K; } @@ -456,6 +432,11 @@ public: /// before using. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Returns the initialization pseudo-symbol, if any. This symbol will also + /// be present in the SymbolFlagsMap for this MaterializationResponsibility + /// object. + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + /// Returns the names of any symbols covered by this /// MaterializationResponsibility object that have queries pending. This /// information can be used to return responsibility for unrequested symbols @@ -489,13 +470,32 @@ public: /// is guaranteed to return Error::success() and can be wrapped with cantFail. Error notifyEmitted(); - /// Adds new symbols to the JITDylib and this responsibility instance. - /// JITDylib entries start out in the materializing state. + /// Attempt to claim responsibility for new definitions. This method can be + /// used to claim responsibility for symbols that are added to a + /// materialization unit during the compilation process (e.g. literal pool + /// symbols). Symbol linkage rules are the same as for symbols that are + /// defined up front: duplicate strong definitions will result in errors. + /// Duplicate weak definitions will be discarded (in which case they will + /// not be added to this responsibility instance). /// /// This method can be used by materialization units that want to add /// additional symbols at materialization time (e.g. stubs, compile /// callbacks, metadata). - Error defineMaterializing(const SymbolFlagsMap &SymbolFlags); + Error defineMaterializing(SymbolFlagsMap SymbolFlags); + + /// Define the given symbols as non-existent, removing it from the symbol + /// table and notifying any pending queries. Queries that lookup up the + /// symbol using the SymbolLookupFlags::WeaklyReferencedSymbol flag will + /// behave as if the symbol had not been matched in the first place. Queries + /// that required this symbol will fail with a missing symbol definition + /// error. + /// + /// This method is intended to support cleanup of special symbols like + /// initializer symbols: Queries using + /// SymbolLookupFlags::WeaklyReferencedSymbol can be used to trigger their + /// emission, and this method can be used to remove them from the JITDylib + /// once materialization is complete. + void defineNonExistent(ArrayRef<SymbolStringPtr> Symbols); /// Notify all not-yet-emitted covered by this MaterializationResponsibility /// instance that an error has occurred. @@ -526,11 +526,18 @@ public: private: /// Create a MaterializationResponsibility for the given JITDylib and /// initial symbols. - MaterializationResponsibility(JITDylib &JD, SymbolFlagsMap SymbolFlags, - VModuleKey K); + MaterializationResponsibility(std::shared_ptr<JITDylib> JD, + SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, VModuleKey K) + : JD(std::move(JD)), SymbolFlags(std::move(SymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert(this->JD && "Cannot initialize with null JD"); + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + } - JITDylib &JD; + std::shared_ptr<JITDylib> JD; SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; }; @@ -543,9 +550,17 @@ private: /// is requested via the lookup method. The JITDylib will call discard if a /// stronger definition is added or already present. class MaterializationUnit { + friend class ExecutionSession; + friend class JITDylib; + public: - MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, VModuleKey K) - : SymbolFlags(std::move(InitalSymbolFlags)), K(std::move(K)) {} + MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, + SymbolStringPtr InitSymbol, VModuleKey K) + : SymbolFlags(std::move(InitalSymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) && + "If set, InitSymbol should appear in InitialSymbolFlags map"); + } virtual ~MaterializationUnit() {} @@ -556,13 +571,13 @@ public: /// Return the set of symbols that this source provides. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } - /// Called by materialization dispatchers (see - /// ExecutionSession::DispatchMaterializationFunction) to trigger - /// materialization of this MaterializationUnit. - void doMaterialize(JITDylib &JD) { - materialize(MaterializationResponsibility(JD, std::move(SymbolFlags), - std::move(K))); - } + /// Returns the initialization symbol for this MaterializationUnit (if any). + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + + /// Implementations of this method should materialize all symbols + /// in the materialzation unit, except for those that have been + /// previously discarded. + virtual void materialize(MaterializationResponsibility R) = 0; /// Called by JITDylibs to notify MaterializationUnits that the given symbol /// has been overridden. @@ -573,15 +588,17 @@ public: protected: SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; private: virtual void anchor(); - /// Implementations of this method should materialize all symbols - /// in the materialzation unit, except for those that have been - /// previously discarded. - virtual void materialize(MaterializationResponsibility R) = 0; + MaterializationResponsibility + createMaterializationResponsibility(std::shared_ptr<JITDylib> JD) { + return MaterializationResponsibility(std::move(JD), std::move(SymbolFlags), + std::move(InitSymbol), K); + } /// Implementations of this method should discard the given symbol /// from the source (e.g. if the source is an LLVM IR Module and the @@ -721,15 +738,6 @@ public: void notifySymbolMetRequiredState(const SymbolStringPtr &Name, JITEvaluatedSymbol Sym); - /// Remove a symbol from the query. This is used to drop weakly referenced - /// symbols that are not found. - void dropSymbol(const SymbolStringPtr &Name) { - assert(ResolvedSymbols.count(Name) && - "Redundant removal of weakly-referenced symbol"); - ResolvedSymbols.erase(Name); - --OutstandingSymbolsCount; - } - /// Returns true if all symbols covered by this query have been /// resolved. bool isComplete() const { return OutstandingSymbolsCount == 0; } @@ -747,6 +755,8 @@ private: void removeQueryDependence(JITDylib &JD, const SymbolStringPtr &Name); + void dropSymbol(const SymbolStringPtr &Name); + bool canStillFail(); void handleFailed(Error Err); @@ -766,9 +776,10 @@ private: /// their addresses may be used as keys for resource management. /// JITDylib state changes must be made via an ExecutionSession to guarantee /// that they are synchronized with respect to other JITDylib operations. -class JITDylib { +class JITDylib : public std::enable_shared_from_this<JITDylib> { friend class AsynchronousSymbolQuery; friend class ExecutionSession; + friend class Platform; friend class MaterializationResponsibility; public: /// Definition generators can be attached to JITDylibs to generate new @@ -817,47 +828,46 @@ public: /// have been added and not yet removed). void removeGenerator(DefinitionGenerator &G); - /// Set the search order to be used when fixing up definitions in JITDylib. - /// This will replace the previous search order, and apply to any symbol + /// Set the link order to be used when fixing up definitions in JITDylib. + /// This will replace the previous link order, and apply to any symbol /// resolutions made for definitions in this JITDylib after the call to - /// setSearchOrder (even if the definition itself was added before the + /// setLinkOrder (even if the definition itself was added before the /// call). /// - /// If SearchThisJITDylibFirst is set, which by default it is, then this - /// JITDylib will add itself to the beginning of the SearchOrder (Clients - /// should *not* put this JITDylib in the list in this case, to avoid - /// redundant lookups). + /// If LinkAgainstThisJITDylibFirst is true (the default) then this JITDylib + /// will add itself to the beginning of the LinkOrder (Clients should not + /// put this JITDylib in the list in this case, to avoid redundant lookups). /// - /// If SearchThisJITDylibFirst is false then the search order will be used as - /// given. The main motivation for this feature is to support deliberate + /// If LinkAgainstThisJITDylibFirst is false then the link order will be used + /// as-is. The primary motivation for this feature is to support deliberate /// shadowing of symbols in this JITDylib by a facade JITDylib. For example, /// the facade may resolve function names to stubs, and the stubs may compile /// lazily by looking up symbols in this dylib. Adding the facade dylib - /// as the first in the search order (instead of this dylib) ensures that + /// as the first in the link order (instead of this dylib) ensures that /// definitions within this dylib resolve to the lazy-compiling stubs, /// rather than immediately materializing the definitions in this dylib. - void setSearchOrder(JITDylibSearchOrder NewSearchOrder, - bool SearchThisJITDylibFirst = true); + void setLinkOrder(JITDylibSearchOrder NewSearchOrder, + bool LinkAgainstThisJITDylibFirst = true); - /// Add the given JITDylib to the search order for definitions in this + /// Add the given JITDylib to the link order for definitions in this /// JITDylib. - void addToSearchOrder(JITDylib &JD, - JITDylibLookupFlags JDLookupFlags = - JITDylibLookupFlags::MatchExportedSymbolsOnly); + void addToLinkOrder(JITDylib &JD, + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); - /// Replace OldJD with NewJD in the search order if OldJD is present. + /// Replace OldJD with NewJD in the link order if OldJD is present. /// Otherwise this operation is a no-op. - void replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, - JITDylibLookupFlags JDLookupFlags = - JITDylibLookupFlags::MatchExportedSymbolsOnly); + void replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); - /// Remove the given JITDylib from the search order for this JITDylib if it is + /// Remove the given JITDylib from the link order for this JITDylib if it is /// present. Otherwise this operation is a no-op. - void removeFromSearchOrder(JITDylib &JD); + void removeFromLinkOrder(JITDylib &JD); - /// Do something with the search order (run under the session lock). + /// Do something with the link order (run under the session lock). template <typename Func> - auto withSearchOrderDo(Func &&F) + auto withLinkOrderDo(Func &&F) -> decltype(F(std::declval<const JITDylibSearchOrder &>())); /// Define all symbols provided by the materialization unit to be part of this @@ -956,11 +966,6 @@ private: JITSymbolFlags getFlags() const { return Flags; } SymbolState getState() const { return static_cast<SymbolState>(State); } - bool isInMaterializationPhase() const { - return getState() == SymbolState::Materializing || - getState() == SymbolState::Resolved; - } - bool hasMaterializerAttached() const { return MaterializerAttached; } bool isPendingRemoval() const { return PendingRemoval; } @@ -1023,7 +1028,7 @@ private: const SymbolStringPtr &DependantName, MaterializingInfo &EmittedMI); - Error defineMaterializing(const SymbolFlagsMap &SymbolFlags); + Expected<SymbolFlagsMap> defineMaterializing(SymbolFlagsMap SymbolFlags); void replace(std::unique_ptr<MaterializationUnit> MU); @@ -1042,11 +1047,41 @@ private: ExecutionSession &ES; std::string JITDylibName; + bool Open = true; SymbolTable Symbols; UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; std::vector<std::unique_ptr<DefinitionGenerator>> DefGenerators; - JITDylibSearchOrder SearchOrder; + JITDylibSearchOrder LinkOrder; +}; + +/// Platforms set up standard symbols and mediate interactions between dynamic +/// initializers (e.g. C++ static constructors) and ExecutionSession state. +/// Note that Platforms do not automatically run initializers: clients are still +/// responsible for doing this. +class Platform { +public: + virtual ~Platform(); + + /// This method will be called outside the session lock each time a JITDylib + /// is created (unless it is created with EmptyJITDylib set) to allow the + /// Platform to install any JITDylib specific standard symbols (e.g + /// __dso_handle). + virtual Error setupJITDylib(JITDylib &JD) = 0; + + /// This method will be called under the ExecutionSession lock each time a + /// MaterializationUnit is added to a JITDylib. + virtual Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) = 0; + + /// This method will be called under the ExecutionSession lock when a + /// VModuleKey is removed. + virtual Error notifyRemoving(JITDylib &JD, VModuleKey K) = 0; + + /// A utility function for looking up initializer symbols. Performs a blocking + /// lookup for the given symbols in each of the given JITDylibs. + static Expected<DenseMap<JITDylib *, SymbolMap>> + lookupInitSymbols(ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms); }; /// An ExecutionSession represents a running JIT program. @@ -1059,8 +1094,9 @@ public: using ErrorReporter = std::function<void(Error)>; /// For dispatching MaterializationUnit::materialize calls. - using DispatchMaterializationFunction = std::function<void( - JITDylib &JD, std::unique_ptr<MaterializationUnit> MU)>; + using DispatchMaterializationFunction = + std::function<void(std::unique_ptr<MaterializationUnit> MU, + MaterializationResponsibility MR)>; /// Construct an ExecutionSession. /// @@ -1073,8 +1109,15 @@ public: /// Returns a shared_ptr to the SymbolStringPool for this ExecutionSession. std::shared_ptr<SymbolStringPool> getSymbolStringPool() const { return SSP; } + /// Set the Platform for this ExecutionSession. + void setPlatform(std::unique_ptr<Platform> P) { this->P = std::move(P); } + + /// Get the Platform for this session. + /// Will return null if no Platform has been set for this ExecutionSession. + Platform *getPlatform() { return P.get(); } + /// Run the given lambda with the session mutex locked. - template <typename Func> auto runSessionLocked(Func &&F) -> decltype(F()) { + template <typename Func> decltype(auto) runSessionLocked(Func &&F) { std::lock_guard<std::recursive_mutex> Lock(SessionMutex); return F(); } @@ -1083,12 +1126,26 @@ public: /// Ownership of JITDylib remains within Execution Session JITDylib *getJITDylibByName(StringRef Name); + /// Add a new bare JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (E.g. by calling getJITDylibByName) if names + /// are based on user input. + /// + /// This call does not install any library code or symbols into the newly + /// created JITDylib. The client is responsible for all configuration. + JITDylib &createBareJITDylib(std::string Name); + /// Add a new JITDylib to this ExecutionSession. /// /// The JITDylib Name is required to be unique. Clients should verify that /// names are not being re-used (e.g. by calling getJITDylibByName) if names /// are based on user input. - JITDylib &createJITDylib(std::string Name); + /// + /// If a Platform is attached then Platform::setupJITDylib will be called to + /// install standard platform symbols (e.g. standard library interposes). + /// If no Platform is attached this call is equivalent to createBareJITDylib. + Expected<JITDylib &> createJITDylib(std::string Name); /// Allocate a module key for a new module to add to the JIT. VModuleKey allocateVModule() { @@ -1172,30 +1229,30 @@ public: /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. - Expected<JITEvaluatedSymbol> lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Symbol); + Expected<JITEvaluatedSymbol> + lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected<JITEvaluatedSymbol> lookup(ArrayRef<JITDylib *> SearchOrder, - SymbolStringPtr Symbol); + Expected<JITEvaluatedSymbol> + lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected<JITEvaluatedSymbol> lookup(ArrayRef<JITDylib *> SearchOrder, - StringRef Symbol); + Expected<JITEvaluatedSymbol> + lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Materialize the given unit. - void dispatchMaterialization(JITDylib &JD, - std::unique_ptr<MaterializationUnit> MU) { - LLVM_DEBUG({ - runSessionLocked([&]() { - dbgs() << "Dispatching " << *MU << " for " << JD.getName() << "\n"; - }); - }); - DispatchMaterialization(JD, std::move(MU)); + void dispatchMaterialization(std::unique_ptr<MaterializationUnit> MU, + MaterializationResponsibility MR) { + assert(MU && "MU must be non-null"); + DEBUG_WITH_TYPE("orc", dumpDispatchInfo(MR.getTargetJITDylib(), *MU)); + DispatchMaterialization(std::move(MU), std::move(MR)); } /// Dump the state of all the JITDylibs in this session. @@ -1207,26 +1264,32 @@ private: } static void - materializeOnCurrentThread(JITDylib &JD, - std::unique_ptr<MaterializationUnit> MU) { - MU->doMaterialize(JD); + materializeOnCurrentThread(std::unique_ptr<MaterializationUnit> MU, + MaterializationResponsibility MR) { + MU->materialize(std::move(MR)); } void runOutstandingMUs(); +#ifndef NDEBUG + void dumpDispatchInfo(JITDylib &JD, MaterializationUnit &MU); +#endif // NDEBUG + mutable std::recursive_mutex SessionMutex; std::shared_ptr<SymbolStringPool> SSP; + std::unique_ptr<Platform> P; VModuleKey LastKey = 0; ErrorReporter ReportError = logErrorsToStdErr; DispatchMaterializationFunction DispatchMaterialization = materializeOnCurrentThread; - std::vector<std::unique_ptr<JITDylib>> JDs; + std::vector<std::shared_ptr<JITDylib>> JDs; // FIXME: Remove this (and runOutstandingMUs) once the linking layer works // with callbacks from asynchronous queries. mutable std::recursive_mutex OutstandingMUsMutex; - std::vector<std::pair<JITDylib *, std::unique_ptr<MaterializationUnit>>> + std::vector<std::pair<std::unique_ptr<MaterializationUnit>, + MaterializationResponsibility>> OutstandingMUs; }; @@ -1239,18 +1302,36 @@ GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { } template <typename Func> -auto JITDylib::withSearchOrderDo(Func &&F) +auto JITDylib::withLinkOrderDo(Func &&F) -> decltype(F(std::declval<const JITDylibSearchOrder &>())) { - return ES.runSessionLocked([&]() { return F(SearchOrder); }); + return ES.runSessionLocked([&]() { return F(LinkOrder); }); } template <typename MaterializationUnitType> Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &&MU) { assert(MU && "Can not define with a null MU"); + + if (MU->getSymbols().empty()) { + // Empty MUs are allowable but pathological, so issue a warning. + DEBUG_WITH_TYPE("orc", { + dbgs() << "Warning: Discarding empty MU " << MU->getName() << " for " + << getName() << "\n"; + }); + return Error::success(); + } else + DEBUG_WITH_TYPE("orc", { + dbgs() << "Defining MU " << MU->getName() << " for " << getName() << "\n"; + }); + return ES.runSessionLocked([&, this]() -> Error { if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1264,10 +1345,27 @@ template <typename MaterializationUnitType> Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) { assert(MU && "Can not define with a null MU"); + if (MU->getSymbols().empty()) { + // Empty MUs are allowable but pathological, so issue a warning. + DEBUG_WITH_TYPE("orc", { + dbgs() << "Warning: Discarding empty MU " << MU->getName() << getName() + << "\n"; + }); + return Error::success(); + } else + DEBUG_WITH_TYPE("orc", { + dbgs() << "Defining MU " << MU->getName() << " for " << getName() << "\n"; + }); + return ES.runSessionLocked([&, this]() -> Error { if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1300,21 +1398,7 @@ private: SymbolPredicate Allow; }; -/// Mangles symbol names then uniques them in the context of an -/// ExecutionSession. -class MangleAndInterner { -public: - MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); - SymbolStringPtr operator()(StringRef Name); - -private: - ExecutionSession &ES; - const DataLayout &DL; -}; - } // End namespace orc } // End namespace llvm -#undef DEBUG_TYPE // "orc" - #endif // LLVM_EXECUTIONENGINE_ORC_CORE_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h index b2ef29d65ffe..4b4472e0ac4d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h @@ -13,7 +13,11 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H #define LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" #include <memory> #include <string> @@ -23,6 +27,74 @@ class MemoryBuffer; namespace orc { +// --raw_ostream operators for ORC types-- + +/// Render a SymbolStringPtr. +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym); + +/// Render a SymbolNameSet. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols); + +/// Render a SymbolNameVector. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols); + +/// Render an array of SymbolStringPtrs. +raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols); + +/// Render JITSymbolFlags. +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags); + +/// Render a SymbolFlagsMap entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV); + +/// Render a SymbolMap entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV); + +/// Render a SymbolFlagsMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags); + +/// Render a SymbolMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols); + +/// Render a SymbolDependenceMap entry. +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV); + +/// Render a SymbolDependendeMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps); + +/// Render a MaterializationUnit. +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU); + +//// Render a JITDylibLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags); + +/// Rendar a SymbolLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags); + +/// Render a JITDylibLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); + +/// Render a SymbolLookupSet entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV); + +/// Render a SymbolLookupSet. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet); + +/// Render a JITDylibSearchOrder. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder); + +/// Render a SymbolAliasMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); + +/// Render a SymbolState. +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); + +/// Render a LookupKind. +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); + /// A function object that can be used as an ObjectTransformLayer transform /// to dump object files to disk at a specified path. class DumpObjects { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index f7255c5af845..3b824b83b052 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -17,6 +17,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/Object/Archive.h" @@ -104,6 +105,53 @@ iterator_range<CtorDtorIterator> getConstructors(const Module &M); /// array. iterator_range<CtorDtorIterator> getDestructors(const Module &M); +/// This iterator provides a convenient way to iterate over GlobalValues that +/// have initialization effects. +class StaticInitGVIterator { +public: + StaticInitGVIterator() = default; + + StaticInitGVIterator(Module &M) + : I(M.global_values().begin()), E(M.global_values().end()), + ObjFmt(Triple(M.getTargetTriple()).getObjectFormat()) { + if (I != E) { + if (!isStaticInitGlobal(*I)) + moveToNextStaticInitGlobal(); + } else + I = E = Module::global_value_iterator(); + } + + bool operator==(const StaticInitGVIterator &O) const { return I == O.I; } + bool operator!=(const StaticInitGVIterator &O) const { return I != O.I; } + + StaticInitGVIterator &operator++() { + assert(I != E && "Increment past end of range"); + moveToNextStaticInitGlobal(); + return *this; + } + + GlobalValue &operator*() { return *I; } + +private: + bool isStaticInitGlobal(GlobalValue &GV); + void moveToNextStaticInitGlobal() { + ++I; + while (I != E && !isStaticInitGlobal(*I)) + ++I; + if (I == E) + I = E = Module::global_value_iterator(); + } + + Module::global_value_iterator I, E; + Triple::ObjectFormatType ObjFmt; +}; + +/// Create an iterator range over the GlobalValues that contribute to static +/// initialization. +inline iterator_range<StaticInitGVIterator> getStaticInitGVs(Module &M) { + return make_range(StaticInitGVIterator(M), StaticInitGVIterator()); +} + /// Convenience class for recording constructor/destructor names for /// later execution. template <typename JITLayerT> @@ -246,6 +294,22 @@ public: Error enable(JITDylib &JD, MangleAndInterner &Mangler); }; +/// An interface for Itanium __cxa_atexit interposer implementations. +class ItaniumCXAAtExitSupport { +public: + struct AtExitRecord { + void (*F)(void *); + void *Ctx; + }; + + void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle); + void runAtExits(void *DSOHandle); + +private: + std::mutex AtExitsMutex; + DenseMap<void *, std::vector<AtExitRecord>> AtExitRecords; +}; + /// A utility class to expose symbols found via dlsym to the JIT. /// /// If an instance of this class is attached to a JITDylib as a fallback @@ -303,6 +367,14 @@ public: static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> Load(ObjectLayer &L, const char *FileName); + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// or a MachO universal binary containing a static library that is compatible + /// with the given triple. Otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName, const Triple &TT); + /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. /// This call will succeed if the buffer contains a valid archive, otherwise /// it will return an error. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index 52223a83ad42..eb74d283f043 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -29,14 +29,29 @@ namespace orc { class IRCompileLayer : public IRLayer { public: - using CompileFunction = - std::function<Expected<std::unique_ptr<MemoryBuffer>>(Module &)>; + class IRCompiler { + public: + IRCompiler(IRSymbolMapper::ManglingOptions MO) : MO(std::move(MO)) {} + virtual ~IRCompiler(); + const IRSymbolMapper::ManglingOptions &getManglingOptions() const { + return MO; + } + virtual Expected<std::unique_ptr<MemoryBuffer>> operator()(Module &M) = 0; + + protected: + IRSymbolMapper::ManglingOptions &manglingOptions() { return MO; } + + private: + IRSymbolMapper::ManglingOptions MO; + }; using NotifyCompiledFunction = std::function<void(VModuleKey K, ThreadSafeModule TSM)>; IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, - CompileFunction Compile); + std::unique_ptr<IRCompiler> Compile); + + IRCompiler &getCompiler() { return *Compile; } void setNotifyCompiled(NotifyCompiledFunction NotifyCompiled); @@ -45,7 +60,8 @@ public: private: mutable std::mutex IRLayerMutex; ObjectLayer &BaseLayer; - CompileFunction Compile; + std::unique_ptr<IRCompiler> Compile; + const IRSymbolMapper::ManglingOptions *ManglingOpts; NotifyCompiledFunction NotifyCompiled = NotifyCompiledFunction(); }; @@ -90,7 +106,10 @@ public: /// Compile the module, and add the resulting object to the base layer /// along with the given memory manager and symbol resolver. Error addModule(VModuleKey K, std::unique_ptr<Module> M) { - if (auto Err = BaseLayer.addObject(std::move(K), Compile(*M))) + auto Obj = Compile(*M); + if (!Obj) + return Obj.takeError(); + if (auto Err = BaseLayer.addObject(std::move(K), std::move(*Obj))) return Err; if (NotifyCompiled) NotifyCompiled(std::move(K), std::move(M)); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index b71e5b339711..296d74ae6b86 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -28,7 +28,7 @@ namespace orc { class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function<Expected<ThreadSafeModule>( - ThreadSafeModule, const MaterializationResponsibility &R)>; + ThreadSafeModule, MaterializationResponsibility &R)>; IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, TransformFunction Transform = identityTransform); @@ -39,9 +39,8 @@ public: void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; - static ThreadSafeModule - identityTransform(ThreadSafeModule TSM, - const MaterializationResponsibility &R) { + static ThreadSafeModule identityTransform(ThreadSafeModule TSM, + MaterializationResponsibility &R) { return TSM; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index a9ab3a630a64..e0cfd8bf2409 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -15,9 +15,9 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Twine.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/Support/Error.h" #include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" @@ -26,6 +26,7 @@ #include <cassert> #include <cstdint> #include <functional> +#include <future> #include <map> #include <memory> #include <system_error> @@ -42,6 +43,7 @@ class GlobalVariable; class Module; class PointerType; class Triple; +class Twine; class Value; namespace orc { @@ -53,6 +55,13 @@ namespace orc { /// are used by various ORC APIs to support lazy compilation class TrampolinePool { public: + using NotifyLandingResolvedFunction = + unique_function<void(JITTargetAddress) const>; + + using ResolveLandingFunction = unique_function<void( + JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction OnLandingResolved) const>; + virtual ~TrampolinePool() {} /// Get an available trampoline address. @@ -66,18 +75,15 @@ private: /// A trampoline pool for trampolines within the current process. template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool { public: - using GetTrampolineLandingFunction = - std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>; - /// Creates a LocalTrampolinePool with the given RunCallback function. /// Returns an error if this function is unable to correctly allocate, write /// and protect the resolver code block. static Expected<std::unique_ptr<LocalTrampolinePool>> - Create(GetTrampolineLandingFunction GetTrampolineLanding) { + Create(ResolveLandingFunction ResolveLanding) { Error Err = Error::success(); auto LTP = std::unique_ptr<LocalTrampolinePool>( - new LocalTrampolinePool(std::move(GetTrampolineLanding), Err)); + new LocalTrampolinePool(std::move(ResolveLanding), Err)); if (Err) return std::move(Err); @@ -108,13 +114,19 @@ private: static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) { LocalTrampolinePool<ORCABI> *TrampolinePool = static_cast<LocalTrampolinePool *>(TrampolinePoolPtr); - return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>( - reinterpret_cast<uintptr_t>(TrampolineId))); + + std::promise<JITTargetAddress> LandingAddressP; + auto LandingAddressF = LandingAddressP.get_future(); + + TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId), + [&](JITTargetAddress LandingAddress) { + LandingAddressP.set_value(LandingAddress); + }); + return LandingAddressF.get(); } - LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding, - Error &Err) - : GetTrampolineLanding(std::move(GetTrampolineLanding)) { + LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err) + : ResolveLanding(std::move(ResolveLanding)) { ErrorAsOutParameter _(&Err); @@ -128,8 +140,10 @@ private: return; } - ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()), - &reenter, this); + ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()), + pointerToJITTargetAddress(ResolverBlock.base()), + pointerToJITTargetAddress(&reenter), + pointerToJITTargetAddress(this)); EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), sys::Memory::MF_READ | @@ -155,14 +169,14 @@ private: (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / ORCABI::TrampolineSize; - uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base()); - ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(), - NumTrampolines); + char *TrampolineMem = static_cast<char *>(TrampolineBlock.base()); + ORCABI::writeTrampolines( + TrampolineMem, pointerToJITTargetAddress(TrampolineMem), + pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines); for (unsigned I = 0; I < NumTrampolines; ++I) - this->AvailableTrampolines.push_back( - static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>( - TrampolineMem + (I * ORCABI::TrampolineSize)))); + this->AvailableTrampolines.push_back(pointerToJITTargetAddress( + TrampolineMem + (I * ORCABI::TrampolineSize))); if (auto EC = sys::Memory::protectMappedMemory( TrampolineBlock.getMemoryBlock(), @@ -173,7 +187,7 @@ private: return Error::success(); } - GetTrampolineLandingFunction GetTrampolineLanding; + ResolveLandingFunction ResolveLanding; std::mutex LTPMutex; sys::OwningMemoryBlock ResolverBlock; @@ -201,7 +215,7 @@ protected: ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : TP(std::move(TP)), ES(ES), - CallbacksJD(ES.createJITDylib("<Callbacks>")), + CallbacksJD(ES.createBareJITDylib("<Callbacks>")), ErrorHandlerAddress(ErrorHandlerAddress) {} void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { @@ -241,10 +255,14 @@ private: JITTargetAddress ErrorHandlerAddress, Error &Err) : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) { + using NotifyLandingResolvedFunction = + TrampolinePool::NotifyLandingResolvedFunction; + ErrorAsOutParameter _(&Err); auto TP = LocalTrampolinePool<ORCABI>::Create( - [this](JITTargetAddress TrampolineAddr) { - return executeCompileCallback(TrampolineAddr); + [this](JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction NotifyLandingResolved) { + NotifyLandingResolved(executeCompileCallback(TrampolineAddr)); }); if (!TP) { @@ -287,6 +305,61 @@ private: virtual void anchor(); }; +template <typename ORCABI> class LocalIndirectStubsInfo { +public: + LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem) + : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {} + + static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs, + unsigned PageSize) { + auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize); + + assert((ISAS.StubBytes % PageSize == 0) && + "StubBytes is not a page size multiple"); + uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize); + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsAndPtrsMem = + sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + ISAS.StubBytes + PointerAlloc, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) + return errorCodeToError(EC); + + sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes); + auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base()); + auto PtrBlockAddress = + pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes; + + ORCABI::writeIndirectStubsBlock(StubsBlockMem, + pointerToJITTargetAddress(StubsBlockMem), + PtrBlockAddress, ISAS.NumStubs); + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem)); + } + + unsigned getNumStubs() const { return NumStubs; } + + void *getStub(unsigned Idx) const { + return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize; + } + + void **getPtr(unsigned Idx) const { + char *PtrsBase = + static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize; + return reinterpret_cast<void **>(PtrsBase) + Idx; + } + +private: + unsigned NumStubs = 0; + sys::OwningMemoryBlock StubsMem; +}; + /// IndirectStubsManager implementation for the host architecture, e.g. /// OrcX86_64. (See OrcArchitectureSupport.h). template <typename TargetT> @@ -364,13 +437,13 @@ private: unsigned NewStubsRequired = NumStubs - FreeStubs.size(); unsigned NewBlockId = IndirectStubsInfos.size(); - typename TargetT::IndirectStubsInfo ISI; - if (auto Err = - TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr)) - return Err; - for (unsigned I = 0; I < ISI.getNumStubs(); ++I) + auto ISI = + LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize); + if (!ISI) + return ISI.takeError(); + for (unsigned I = 0; I < ISI->getNumStubs(); ++I) FreeStubs.push_back(std::make_pair(NewBlockId, I)); - IndirectStubsInfos.push_back(std::move(ISI)); + IndirectStubsInfos.push_back(std::move(*ISI)); return Error::success(); } @@ -379,12 +452,13 @@ private: auto Key = FreeStubs.back(); FreeStubs.pop_back(); *IndirectStubsInfos[Key.first].getPtr(Key.second) = - reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr)); + jitTargetAddressToPointer<void *>(InitAddr); StubIndexes[StubName] = std::make_pair(Key, StubFlags); } + unsigned PageSize = sys::Process::getPageSizeEstimate(); std::mutex StubsMutex; - std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos; + std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos; using StubKey = std::pair<uint16_t, uint16_t>; std::vector<StubKey> FreeStubs; StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h index 4f2f55770996..c4109a8de82e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h @@ -25,6 +25,9 @@ #include <vector> namespace llvm { + +class raw_ostream; + namespace orc { /// A utility class for building TargetMachines for JITs. @@ -136,6 +139,12 @@ public: /// Access Triple. const Triple &getTargetTriple() const { return TT; } +#ifndef NDEBUG + /// Debug-dump a JITTargetMachineBuilder. + friend raw_ostream &operator<<(raw_ostream &OS, + const JITTargetMachineBuilder &JTMB); +#endif + private: Triple TT; std::string CPU; @@ -143,7 +152,7 @@ private: TargetOptions Options; Optional<Reloc::Model> RM; Optional<CodeModel::Model> CM; - CodeGenOpt::Level OptLevel = CodeGenOpt::None; + CodeGenOpt::Level OptLevel = CodeGenOpt::Default; }; } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index c048ff3d5522..96f8e169e7dc 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ThreadPool.h" namespace llvm { @@ -35,8 +36,22 @@ class LLLazyJITBuilderState; class LLJIT { template <typename, typename, typename> friend class LLJITBuilderSetters; + friend void setUpGenericLLVMIRPlatform(LLJIT &J); + public: - static Expected<std::unique_ptr<LLJIT>> Create(LLJITBuilderState &S); + /// Initializer support for LLJIT. + class PlatformSupport { + public: + virtual ~PlatformSupport(); + + virtual Error initialize(JITDylib &JD) = 0; + + virtual Error deinitialize(JITDylib &JD) = 0; + + protected: + static void setInitTransform(LLJIT &J, + IRTransformLayer::TransformFunction T); + }; /// Destruct this instance. If a multi-threaded instance, waits for all /// compile threads to complete. @@ -45,11 +60,14 @@ public: /// Returns the ExecutionSession for this instance. ExecutionSession &getExecutionSession() { return *ES; } + /// Returns a reference to the triple for this instance. + const Triple &getTargetTriple() const { return TT; } + /// Returns a reference to the DataLayout for this instance. const DataLayout &getDataLayout() const { return DL; } /// Returns a reference to the JITDylib representing the JIT'd main program. - JITDylib &getMainJITDylib() { return Main; } + JITDylib &getMainJITDylib() { return *Main; } /// Returns the JITDylib with the given name, or nullptr if no JITDylib with /// that name exists. @@ -63,19 +81,32 @@ public: /// input or elsewhere in the environment then the client should check /// (e.g. by calling getJITDylibByName) that the given name is not already in /// use. - JITDylib &createJITDylib(std::string Name) { + Expected<JITDylib &> createJITDylib(std::string Name) { return ES->createJITDylib(std::move(Name)); } - /// Convenience method for defining an absolute symbol. - Error defineAbsolute(StringRef Name, JITEvaluatedSymbol Address); + /// A convenience method for defining MUs in LLJIT's Main JITDylib. This can + /// be useful for succinctly defining absolute symbols, aliases and + /// re-exports. + template <typename MUType> + Error define(std::unique_ptr<MUType> &&MU) { + return Main->define(std::move(MU)); + } + + /// A convenience method for defining MUs in LLJIT's Main JITDylib. This can + /// be usedful for succinctly defining absolute symbols, aliases and + /// re-exports. + template <typename MUType> + Error define(std::unique_ptr<MUType> &MU) { + return Main->define(MU); + } /// Adds an IR module to the given JITDylib. Error addIRModule(JITDylib &JD, ThreadSafeModule TSM); /// Adds an IR module to the Main JITDylib. Error addIRModule(ThreadSafeModule TSM) { - return addIRModule(Main, std::move(TSM)); + return addIRModule(*Main, std::move(TSM)); } /// Adds an object file to the given JITDylib. @@ -83,19 +114,26 @@ public: /// Adds an object file to the given JITDylib. Error addObjectFile(std::unique_ptr<MemoryBuffer> Obj) { - return addObjectFile(Main, std::move(Obj)); + return addObjectFile(*Main, std::move(Obj)); } /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to /// look up symbols based on their IR name use the lookup function instead). Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, - StringRef Name); + SymbolStringPtr Name); + + /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to + /// look up symbols based on their IR name use the lookup function instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, + StringRef Name) { + return lookupLinkerMangled(JD, ES->intern(Name)); + } /// Look up a symbol in the main JITDylib by the symbol's linker-mangled name /// (to look up symbols based on their IR name use the lookup function /// instead). Expected<JITEvaluatedSymbol> lookupLinkerMangled(StringRef Name) { - return lookupLinkerMangled(Main, Name); + return lookupLinkerMangled(*Main, Name); } /// Look up a symbol in JITDylib JD based on its IR symbol name. @@ -105,14 +143,36 @@ public: /// Look up a symbol in the main JITDylib based on its IR symbol name. Expected<JITEvaluatedSymbol> lookup(StringRef UnmangledName) { - return lookup(Main, UnmangledName); + return lookup(*Main, UnmangledName); + } + + /// Set the PlatformSupport instance. + void setPlatformSupport(std::unique_ptr<PlatformSupport> PS) { + this->PS = std::move(PS); } - /// Runs all not-yet-run static constructors. - Error runConstructors() { return CtorRunner.run(); } + /// Get the PlatformSupport instance. + PlatformSupport *getPlatformSupport() { return PS.get(); } + + /// Run the initializers for the given JITDylib. + Error initialize(JITDylib &JD) { + DEBUG_WITH_TYPE("orc", { + dbgs() << "LLJIT running initializers for JITDylib \"" << JD.getName() + << "\"\n"; + }); + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->initialize(JD); + } - /// Runs all not-yet-run static destructors. - Error runDestructors() { return DtorRunner.run(); } + /// Run the deinitializers for the given JITDylib. + Error deinitialize(JITDylib &JD) { + DEBUG_WITH_TYPE("orc", { + dbgs() << "LLJIT running deinitializers for JITDylib \"" << JD.getName() + << "\"\n"; + }); + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->deinitialize(JD); + } /// Returns a reference to the ObjLinkingLayer ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } @@ -120,33 +180,48 @@ public: /// Returns a reference to the object transform layer. ObjectTransformLayer &getObjTransformLayer() { return ObjTransformLayer; } + /// Returns a reference to the IR transform layer. + IRTransformLayer &getIRTransformLayer() { return *TransformLayer; } + + /// Returns a reference to the IR compile layer. + IRCompileLayer &getIRCompileLayer() { return *CompileLayer; } + + /// Returns a linker-mangled version of UnmangledName. + std::string mangle(StringRef UnmangledName) const; + + /// Returns an interned, linker-mangled version of UnmangledName. + SymbolStringPtr mangleAndIntern(StringRef UnmangledName) const { + return ES->intern(mangle(UnmangledName)); + } + protected: static std::unique_ptr<ObjectLayer> createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); - static Expected<IRCompileLayer::CompileFunction> + static Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> createCompileFunction(LLJITBuilderState &S, JITTargetMachineBuilder JTMB); /// Create an LLJIT instance with a single compile thread. LLJIT(LLJITBuilderState &S, Error &Err); - std::string mangle(StringRef UnmangledName); - Error applyDataLayout(Module &M); void recordCtorDtors(Module &M); std::unique_ptr<ExecutionSession> ES; - JITDylib &Main; + std::unique_ptr<PlatformSupport> PS; + + JITDylib *Main = nullptr; DataLayout DL; + Triple TT; std::unique_ptr<ThreadPool> CompileThreads; std::unique_ptr<ObjectLayer> ObjLinkingLayer; ObjectTransformLayer ObjTransformLayer; std::unique_ptr<IRCompileLayer> CompileLayer; - - CtorDtorRunner CtorRunner, DtorRunner; + std::unique_ptr<IRTransformLayer> TransformLayer; + std::unique_ptr<IRTransformLayer> InitHelperTransformLayer; }; /// An extended version of LLJIT that supports lazy function-at-a-time @@ -156,12 +231,6 @@ class LLLazyJIT : public LLJIT { public: - /// Set an IR transform (e.g. pass manager pipeline) to run on each function - /// when it is compiled. - void setLazyCompileTransform(IRTransformLayer::TransformFunction Transform) { - TransformLayer->setTransform(std::move(Transform)); - } - /// Sets the partition function. void setPartitionFunction(CompileOnDemandLayer::PartitionFunction Partition) { @@ -173,7 +242,7 @@ public: /// Add a module to be lazily compiled to the main JITDylib. Error addLazyIRModule(ThreadSafeModule M) { - return addLazyIRModule(Main, std::move(M)); + return addLazyIRModule(*Main, std::move(M)); } private: @@ -182,7 +251,6 @@ private: LLLazyJIT(LLLazyJITBuilderState &S, Error &Err); std::unique_ptr<LazyCallThroughManager> LCTMgr; - std::unique_ptr<IRTransformLayer> TransformLayer; std::unique_ptr<CompileOnDemandLayer> CODLayer; }; @@ -192,13 +260,17 @@ public: ExecutionSession &, const Triple &TT)>; using CompileFunctionCreator = - std::function<Expected<IRCompileLayer::CompileFunction>( + std::function<Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>( JITTargetMachineBuilder JTMB)>; + using PlatformSetupFunction = std::function<Error(LLJIT &J)>; + std::unique_ptr<ExecutionSession> ES; Optional<JITTargetMachineBuilder> JTMB; + Optional<DataLayout> DL; ObjectLinkingLayerCreator CreateObjectLinkingLayer; CompileFunctionCreator CreateCompileFunction; + PlatformSetupFunction SetUpPlatform; unsigned NumCompileThreads = 0; /// Called prior to JIT class construcion to fix up defaults. @@ -208,6 +280,13 @@ public: template <typename JITType, typename SetterImpl, typename State> class LLJITBuilderSetters { public: + + /// Set an ExecutionSession for this instance. + SetterImpl &setExecutionSession(std::unique_ptr<ExecutionSession> ES) { + impl().ES = std::move(ES); + return impl(); + } + /// Set the JITTargetMachineBuilder for this instance. /// /// If this method is not called, JITTargetMachineBuilder::detectHost will be @@ -223,6 +302,13 @@ public: return impl().JTMB; } + /// Set a DataLayout for this instance. If no data layout is specified then + /// the target's default data layout will be used. + SetterImpl &setDataLayout(Optional<DataLayout> DL) { + impl().DL = std::move(DL); + return impl(); + } + /// Set an ObjectLinkingLayer creation function. /// /// If this method is not called, a default creation function will be used @@ -245,6 +331,16 @@ public: return impl(); } + /// Set up an PlatformSetupFunction. + /// + /// If this method is not called then setUpGenericLLVMIRPlatform + /// will be used to configure the JIT's platform support. + SetterImpl & + setPlatformSetUp(LLJITBuilderState::PlatformSetupFunction SetUpPlatform) { + impl().SetUpPlatform = std::move(SetUpPlatform); + return impl(); + } + /// Set the number of compile threads to use. /// /// If set to zero, compilation will be performed on the execution thread when @@ -333,6 +429,26 @@ class LLLazyJITBuilder public LLLazyJITBuilderSetters<LLLazyJIT, LLLazyJITBuilder, LLLazyJITBuilderState> {}; +/// Configure the LLJIT instance to scrape modules for llvm.global_ctors and +/// llvm.global_dtors variables and (if present) build initialization and +/// deinitialization functions. Platform specific initialization configurations +/// should be preferred where available. +void setUpGenericLLVMIRPlatform(LLJIT &J); + +/// Configure the LLJIT instance to use MachOPlatform support. +/// +/// Warning: MachOPlatform *requires* that LLJIT be configured to use +/// ObjectLinkingLayer (default on platforms supported by JITLink). If +/// MachOPlatform is used with RTDyldObjectLinkingLayer it will result in +/// undefined behavior). +/// +/// MachOPlatform installs an ObjectLinkingLayer plugin to scrape initializers +/// from the __mod_inits section. It also provides interposes for the dlfcn +/// functions (dlopen, dlclose, dlsym, dlerror) that work for JITDylibs as +/// well as regular libraries (JITDylibs will be preferenced, so make sure +/// your JITDylib names do not shadow any real library paths). +Error setUpMachOPlatform(LLJIT &J); + } // End namespace orc } // End namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h index 8f9bd704395e..e843d0f56245 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_LAYER_H #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "llvm/IR/Module.h" #include "llvm/Support/MemoryBuffer.h" @@ -21,15 +22,62 @@ namespace llvm { namespace orc { +/// IRMaterializationUnit is a convenient base class for MaterializationUnits +/// wrapping LLVM IR. Represents materialization responsibility for all symbols +/// in the given module. If symbols are overridden by other definitions, then +/// their linkage is changed to available-externally. +class IRMaterializationUnit : public MaterializationUnit { +public: + using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; + + /// Create an IRMaterializationLayer. Scans the module to build the + /// SymbolFlags and SymbolToDefinition maps. + IRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K); + + /// Create an IRMaterializationLayer from a module, and pre-existing + /// SymbolFlags and SymbolToDefinition maps. The maps must provide + /// entries for each definition in M. + /// This constructor is useful for delegating work from one + /// IRMaterializationUnit to another. + IRMaterializationUnit(ThreadSafeModule TSM, VModuleKey K, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol, + SymbolNameToDefinitionMap SymbolToDefinition); + + /// Return the ModuleIdentifier as the name for this MaterializationUnit. + StringRef getName() const override; + + /// Return a reference to the contained ThreadSafeModule. + const ThreadSafeModule &getModule() const { return TSM; } + +protected: + ThreadSafeModule TSM; + SymbolNameToDefinitionMap SymbolToDefinition; + +private: + static SymbolStringPtr getInitSymbol(ExecutionSession &ES, + const ThreadSafeModule &TSM); + + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; +}; + /// Interface for layers that accept LLVM IR. class IRLayer { public: - IRLayer(ExecutionSession &ES); + IRLayer(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions *&MO) + : ES(ES), MO(MO) {} + virtual ~IRLayer(); /// Returns the ExecutionSession for this layer. ExecutionSession &getExecutionSession() { return ES; } + /// Get the mangling options for this layer. + const IRSymbolMapper::ManglingOptions *&getManglingOptions() const { + return MO; + } + /// Sets the CloneToNewContextOnEmit flag (false by default). /// /// When set, IR modules added to this layer will be cloned on to a new @@ -57,49 +105,16 @@ public: private: bool CloneToNewContextOnEmit = false; ExecutionSession &ES; -}; - -/// IRMaterializationUnit is a convenient base class for MaterializationUnits -/// wrapping LLVM IR. Represents materialization responsibility for all symbols -/// in the given module. If symbols are overridden by other definitions, then -/// their linkage is changed to available-externally. -class IRMaterializationUnit : public MaterializationUnit { -public: - using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; - - /// Create an IRMaterializationLayer. Scans the module to build the - /// SymbolFlags and SymbolToDefinition maps. - IRMaterializationUnit(ExecutionSession &ES, ThreadSafeModule TSM, - VModuleKey K); - - /// Create an IRMaterializationLayer from a module, and pre-existing - /// SymbolFlags and SymbolToDefinition maps. The maps must provide - /// entries for each definition in M. - /// This constructor is useful for delegating work from one - /// IRMaterializationUnit to another. - IRMaterializationUnit(ThreadSafeModule TSM, VModuleKey K, - SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition); - - /// Return the ModuleIdentifier as the name for this MaterializationUnit. - StringRef getName() const override; - - const ThreadSafeModule &getModule() const { return TSM; } - -protected: - ThreadSafeModule TSM; - SymbolNameToDefinitionMap SymbolToDefinition; - -private: - void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; + const IRSymbolMapper::ManglingOptions *&MO; }; /// MaterializationUnit that materializes modules by calling the 'emit' method /// on the given IRLayer. class BasicIRLayerMaterializationUnit : public IRMaterializationUnit { public: - BasicIRLayerMaterializationUnit(IRLayer &L, VModuleKey K, - ThreadSafeModule TSM); + BasicIRLayerMaterializationUnit(IRLayer &L, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K); private: @@ -140,7 +155,8 @@ public: BasicObjectLayerMaterializationUnit(ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O, - SymbolFlagsMap SymbolFlags); + SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol); /// Return the buffer's identifier as the name for this MaterializationUnit. StringRef getName() const override; @@ -154,12 +170,6 @@ private: std::unique_ptr<MemoryBuffer> O; }; -/// Returns a SymbolFlagsMap for the object file represented by the given -/// buffer, or an error if the buffer does not contain a valid object file. -// FIXME: Maybe move to Core.h? -Expected<SymbolFlagsMap> getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer); - } // End namespace orc } // End namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index b67a9feed523..84f5e0350c2e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -78,7 +78,7 @@ private: // RuntimeDyld that did the lookup), so just return a nullptr here. return nullptr; case Emitted: - return B.findSymbolIn(K, Name, ExportedSymbolsOnly); + return B.findSymbolIn(K, std::string(Name), ExportedSymbolsOnly); } llvm_unreachable("Invalid emit-state."); } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h index 311ed59b1549..0d3ccecdf121 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -16,6 +16,7 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_LAZYREEXPORTS_H #define LLVM_EXECUTIONENGINE_ORC_LAZYREEXPORTS_H +#include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/Speculation.h" @@ -36,73 +37,48 @@ namespace orc { /// function. class LazyCallThroughManager { public: - /// Clients will want to take some action on first resolution, e.g. updating - /// a stub pointer. Instances of this class can be used to implement this. - class NotifyResolvedFunction { - public: - virtual ~NotifyResolvedFunction() {} - - /// Called the first time a lazy call through is executed and the target - /// symbol resolved. - virtual Error operator()(JITDylib &SourceJD, - const SymbolStringPtr &SymbolName, - JITTargetAddress ResolvedAddr) = 0; - - private: - virtual void anchor(); - }; - - template <typename NotifyResolvedImpl> - class NotifyResolvedFunctionImpl : public NotifyResolvedFunction { - public: - NotifyResolvedFunctionImpl(NotifyResolvedImpl NotifyResolved) - : NotifyResolved(std::move(NotifyResolved)) {} - Error operator()(JITDylib &SourceJD, const SymbolStringPtr &SymbolName, - JITTargetAddress ResolvedAddr) { - return NotifyResolved(SourceJD, SymbolName, ResolvedAddr); - } - - private: - NotifyResolvedImpl NotifyResolved; - }; - - /// Create a shared NotifyResolvedFunction from a given type that is - /// callable with the correct signature. - template <typename NotifyResolvedImpl> - static std::unique_ptr<NotifyResolvedFunction> - createNotifyResolvedFunction(NotifyResolvedImpl NotifyResolved) { - return std::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( - std::move(NotifyResolved)); - } + using NotifyResolvedFunction = + unique_function<Error(JITTargetAddress ResolvedAddr)>; // Return a free call-through trampoline and bind it to look up and call // through to the given symbol. - Expected<JITTargetAddress> getCallThroughTrampoline( - JITDylib &SourceJD, SymbolStringPtr SymbolName, - std::shared_ptr<NotifyResolvedFunction> NotifyResolved); + Expected<JITTargetAddress> + getCallThroughTrampoline(JITDylib &SourceJD, SymbolStringPtr SymbolName, + NotifyResolvedFunction NotifyResolved); + + void resolveTrampolineLandingAddress( + JITTargetAddress TrampolineAddr, + TrampolinePool::NotifyLandingResolvedFunction NotifyLandingResolved); + + virtual ~LazyCallThroughManager() = default; protected: + using NotifyLandingResolvedFunction = + TrampolinePool::NotifyLandingResolvedFunction; + LazyCallThroughManager(ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddr, - std::unique_ptr<TrampolinePool> TP); + JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP); - JITTargetAddress callThroughToSymbol(JITTargetAddress TrampolineAddr); + struct ReexportsEntry { + JITDylib *SourceJD; + SymbolStringPtr SymbolName; + }; - void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { - this->TP = std::move(TP); - } + JITTargetAddress reportCallThroughError(Error Err); + Expected<ReexportsEntry> findReexport(JITTargetAddress TrampolineAddr); + Error notifyResolved(JITTargetAddress TrampolineAddr, + JITTargetAddress ResolvedAddr); + void setTrampolinePool(TrampolinePool &TP) { this->TP = &TP; } private: - using ReexportsMap = - std::map<JITTargetAddress, std::pair<JITDylib *, SymbolStringPtr>>; + using ReexportsMap = std::map<JITTargetAddress, ReexportsEntry>; - using NotifiersMap = - std::map<JITTargetAddress, std::shared_ptr<NotifyResolvedFunction>>; + using NotifiersMap = std::map<JITTargetAddress, NotifyResolvedFunction>; std::mutex LCTMMutex; ExecutionSession &ES; JITTargetAddress ErrorHandlerAddr; - std::unique_ptr<TrampolinePool> TP; + TrampolinePool *TP = nullptr; ReexportsMap Reexports; NotifiersMap Notifiers; }; @@ -110,23 +86,31 @@ private: /// A lazy call-through manager that builds trampolines in the current process. class LocalLazyCallThroughManager : public LazyCallThroughManager { private: + using NotifyTargetResolved = unique_function<void(JITTargetAddress)>; + LocalLazyCallThroughManager(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) : LazyCallThroughManager(ES, ErrorHandlerAddr, nullptr) {} template <typename ORCABI> Error init() { auto TP = LocalTrampolinePool<ORCABI>::Create( - [this](JITTargetAddress TrampolineAddr) { - return callThroughToSymbol(TrampolineAddr); + [this](JITTargetAddress TrampolineAddr, + TrampolinePool::NotifyLandingResolvedFunction + NotifyLandingResolved) { + resolveTrampolineLandingAddress(TrampolineAddr, + std::move(NotifyLandingResolved)); }); if (!TP) return TP.takeError(); - setTrampolinePool(std::move(*TP)); + this->TP = std::move(*TP); + setTrampolinePool(*this->TP); return Error::success(); } + std::unique_ptr<TrampolinePool> TP; + public: /// Create a LocalLazyCallThroughManager using the given ABI. See /// createLocalLazyCallThroughManager. @@ -173,8 +157,6 @@ private: IndirectStubsManager &ISManager; JITDylib &SourceJD; SymbolAliasMap CallableAliases; - std::shared_ptr<LazyCallThroughManager::NotifyResolvedFunction> - NotifyResolved; ImplSymbolMap *AliaseeTable; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h b/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h index 148e260c9569..b20202a49ef6 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -73,17 +73,13 @@ private: /// function objects. template <typename GetResponsibilitySetFn, typename LookupFn> std::unique_ptr<LambdaSymbolResolver< - typename std::remove_cv< - typename std::remove_reference<GetResponsibilitySetFn>::type>::type, - typename std::remove_cv< - typename std::remove_reference<LookupFn>::type>::type>> + std::remove_cv_t<std::remove_reference_t<GetResponsibilitySetFn>>, + std::remove_cv_t<std::remove_reference_t<LookupFn>>>> createSymbolResolver(GetResponsibilitySetFn &&GetResponsibilitySet, LookupFn &&Lookup) { using LambdaSymbolResolverImpl = LambdaSymbolResolver< - typename std::remove_cv< - typename std::remove_reference<GetResponsibilitySetFn>::type>::type, - typename std::remove_cv< - typename std::remove_reference<LookupFn>::type>::type>; + std::remove_cv_t<std::remove_reference_t<GetResponsibilitySetFn>>, + std::remove_cv_t<std::remove_reference_t<LookupFn>>>; return std::make_unique<LambdaSymbolResolverImpl>( std::forward<GetResponsibilitySetFn>(GetResponsibilitySet), std::forward<LookupFn>(Lookup)); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h new file mode 100644 index 000000000000..15fe079eccaf --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -0,0 +1,161 @@ +//===-- MachOPlatform.h - Utilities for executing MachO in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for executing JIT'd MachO in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include <future> +#include <thread> +#include <vector> + +namespace llvm { +namespace orc { + +/// Enable registration of JIT'd ObjC classes and selectors. +Error enableObjCRegistration(const char *PathToLibObjC); +bool objCRegistrationEnabled(); + +class MachOJITDylibInitializers { +public: + struct SectionExtent { + SectionExtent() = default; + SectionExtent(JITTargetAddress Address, uint64_t NumPtrs) + : Address(Address), NumPtrs(NumPtrs) {} + JITTargetAddress Address = 0; + uint64_t NumPtrs = 0; + }; + + using RawPointerSectionList = std::vector<SectionExtent>; + + void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { + this->ObjCImageInfoAddr = ObjCImageInfoAddr; + } + + void addModInitsSection(SectionExtent ModInit) { + ModInitSections.push_back(std::move(ModInit)); + } + + const RawPointerSectionList &getModInitsSections() const { + return ModInitSections; + } + + void addObjCSelRefsSection(SectionExtent ObjCSelRefs) { + ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); + } + + const RawPointerSectionList &getObjCSelRefsSections() const { + return ObjCSelRefsSections; + } + + void addObjCClassListSection(SectionExtent ObjCClassList) { + ObjCClassListSections.push_back(std::move(ObjCClassList)); + } + + const RawPointerSectionList &getObjCClassListSections() const { + return ObjCClassListSections; + } + + void runModInits() const; + void registerObjCSelectors() const; + Error registerObjCClasses() const; + +private: + + JITTargetAddress ObjCImageInfoAddr; + RawPointerSectionList ModInitSections; + RawPointerSectionList ObjCSelRefsSections; + RawPointerSectionList ObjCClassListSections; +}; + +class MachOJITDylibDeinitializers {}; + +/// Mediates between MachO initialization and ExecutionSession state. +class MachOPlatform : public Platform { +public: + using InitializerSequence = + std::vector<std::pair<JITDylib *, MachOJITDylibInitializers>>; + + using DeinitializerSequence = + std::vector<std::pair<JITDylib *, MachOJITDylibDeinitializers>>; + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr<MemoryBuffer> StandardSymbolsObject); + + ExecutionSession &getExecutionSession() const { return ES; } + + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override; + + Expected<InitializerSequence> getInitializerSequence(JITDylib &JD); + + Expected<DeinitializerSequence> getDeinitializerSequence(JITDylib &JD); + +private: + // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, + // __objc_classlist and __sel_ref sections and records their extents so that + // they can be run in the target process. + class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + public: + InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) override; + + LocalDependenciesMap getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) override; + + private: + using InitSymbolDepMap = + DenseMap<MaterializationResponsibility *, JITLinkSymbolVector>; + + void preserveInitSectionIfPresent(JITLinkSymbolVector &Syms, + jitlink::LinkGraph &G, + StringRef SectionName); + + Error processObjCImageInfo(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + std::mutex InitScraperMutex; + MachOPlatform &MP; + DenseMap<JITDylib *, std::pair<uint32_t, uint32_t>> ObjCImageInfos; + InitSymbolDepMap InitSymbolDeps; + }; + + static std::vector<JITDylib *> getDFSLinkOrder(JITDylib &JD); + + void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList); + + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + std::unique_ptr<MemoryBuffer> StandardSymbolsObject; + + DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols; + + // InitSeqs gets its own mutex to avoid locking the whole session when + // aggregating data from the jitlink. + std::mutex InitSeqsMutex; + DenseMap<JITDylib *, MachOJITDylibInitializers> InitSeqs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h new file mode 100644 index 000000000000..e0f770a601fb --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h @@ -0,0 +1,66 @@ +//===------ Mangling.h -- Name Mangling Utilities for ORC -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Name mangling utilities for ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MANGLING_H +#define LLVM_EXECUTIONENGINE_ORC_MANGLING_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// Mangles symbol names then uniques them in the context of an +/// ExecutionSession. +class MangleAndInterner { +public: + MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); + SymbolStringPtr operator()(StringRef Name); + +private: + ExecutionSession &ES; + const DataLayout &DL; +}; + +/// Maps IR global values to their linker symbol names / flags. +/// +/// This utility can be used when adding new IR globals in the JIT. +class IRSymbolMapper { +public: + struct ManglingOptions { + bool EmulatedTLS = false; + }; + + using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; + + /// Add mangled symbols for the given GlobalValues to SymbolFlags. + /// If a SymbolToDefinitionMap pointer is supplied then it will be populated + /// with Name-to-GlobalValue* mappings. Note that this mapping is not + /// necessarily one-to-one: thread-local GlobalValues, for example, may + /// produce more than one symbol, in which case the map will contain duplicate + /// values. + static void add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef<GlobalValue *> GVs, SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition = nullptr); +}; + +/// Returns a SymbolFlagsMap for the object file represented by the given +/// buffer, or an error if the buffer does not contain a valid object file. +Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MANGLING_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index 50d25f18891e..2bfe3b001709 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -35,6 +35,7 @@ namespace llvm { namespace jitlink { class EHFrameRegistrar; +class Symbol; } // namespace jitlink namespace object { @@ -59,10 +60,14 @@ public: /// configured. class Plugin { public: + using JITLinkSymbolVector = std::vector<const jitlink::Symbol *>; + using LocalDependenciesMap = DenseMap<SymbolStringPtr, JITLinkSymbolVector>; + virtual ~Plugin(); virtual void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, jitlink::PassConfiguration &Config) {} + virtual void notifyLoaded(MaterializationResponsibility &MR) {} virtual Error notifyEmitted(MaterializationResponsibility &MR) { return Error::success(); @@ -71,6 +76,15 @@ public: return Error::success(); } virtual Error notifyRemovingAllModules() { return Error::success(); } + + /// Return any dependencies that synthetic symbols (e.g. init symbols) + /// have on locally scoped jitlink::Symbols. This is used by the + /// ObjectLinkingLayer to update the dependencies for the synthetic + /// symbols. + virtual LocalDependenciesMap + getSyntheticSymbolLocalDependencies(MaterializationResponsibility &MR) { + return LocalDependenciesMap(); + } }; using ReturnObjectBufferFunction = @@ -170,6 +184,7 @@ private: size_t Size; }; + std::mutex EHFramePluginMutex; jitlink::EHFrameRegistrar &Registrar; DenseMap<MaterializationResponsibility *, EHFrameRange> InProcessLinks; DenseMap<VModuleKey, EHFrameRange> TrackedEHFrameRanges; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h index 2e58ddd75d31..5061c15cf4c9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -20,13 +20,33 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Memory.h" +#include "llvm/Support/MathExtras.h" #include <algorithm> #include <cstdint> namespace llvm { namespace orc { +struct IndirectStubsAllocationSizes { + uint64_t StubBytes = 0; + uint64_t PointerBytes = 0; + unsigned NumStubs = 0; +}; + +template <typename ORCABI> +IndirectStubsAllocationSizes +getIndirectStubsBlockSizes(unsigned MinStubs, unsigned RoundToMultipleOf = 0) { + assert( + (RoundToMultipleOf == 0 || (RoundToMultipleOf % ORCABI::StubSize == 0)) && + "RoundToMultipleOf is not a multiple of stub size"); + uint64_t StubBytes = MinStubs * ORCABI::StubSize; + if (RoundToMultipleOf) + StubBytes = alignTo(StubBytes, RoundToMultipleOf); + unsigned NumStubs = StubBytes / ORCABI::StubSize; + uint64_t PointerBytes = NumStubs * ORCABI::PointerSize; + return {StubBytes, PointerBytes, NumStubs}; +} + /// Generic ORC ABI support. /// /// This class can be substituted as the target architecture support class for @@ -35,113 +55,72 @@ namespace orc { /// will result in execution of an llvm_unreachable. class OrcGenericABI { public: - static const unsigned PointerSize = sizeof(uintptr_t); - static const unsigned TrampolineSize = 1; - static const unsigned ResolverCodeSize = 1; - - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); - - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr) { + static constexpr unsigned PointerSize = sizeof(uintptr_t); + static constexpr unsigned TrampolineSize = 1; + static constexpr unsigned StubSize = 1; + static constexpr unsigned StubToPointerMaxDisplacement = 1; + static constexpr unsigned ResolverCodeSize = 1; + + static void writeResolverCode(char *ResolveWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { llvm_unreachable("writeResolverCode is not supported by the generic host " "support class"); } - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddr, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { llvm_unreachable("writeTrampolines is not supported by the generic host " "support class"); } - class IndirectStubsInfo { - public: - const static unsigned StubSize = 1; - - unsigned getNumStubs() const { llvm_unreachable("Not supported"); } - void *getStub(unsigned Idx) const { llvm_unreachable("Not supported"); } - void **getPtr(unsigned Idx) const { llvm_unreachable("Not supported"); } - }; - - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, void *InitialPtrVal) { - llvm_unreachable("emitIndirectStubsBlock is not supported by the generic " - "host support class"); - } -}; - -/// Provide information about stub blocks generated by the -/// makeIndirectStubsBlock function. -template <unsigned StubSizeVal> class GenericIndirectStubsInfo { -public: - const static unsigned StubSize = StubSizeVal; - - GenericIndirectStubsInfo() = default; - GenericIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem) - : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {} - GenericIndirectStubsInfo(GenericIndirectStubsInfo &&Other) - : NumStubs(Other.NumStubs), StubsMem(std::move(Other.StubsMem)) { - Other.NumStubs = 0; - } - - GenericIndirectStubsInfo &operator=(GenericIndirectStubsInfo &&Other) { - NumStubs = Other.NumStubs; - Other.NumStubs = 0; - StubsMem = std::move(Other.StubsMem); - return *this; - } - - /// Number of stubs in this block. - unsigned getNumStubs() const { return NumStubs; } - - /// Get a pointer to the stub at the given index, which must be in - /// the range 0 .. getNumStubs() - 1. - void *getStub(unsigned Idx) const { - return static_cast<char *>(StubsMem.base()) + Idx * StubSize; + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + llvm_unreachable( + "writeIndirectStubsBlock is not supported by the generic host " + "support class"); } - - /// Get a pointer to the implementation-pointer at the given index, - /// which must be in the range 0 .. getNumStubs() - 1. - void **getPtr(unsigned Idx) const { - char *PtrsBase = static_cast<char *>(StubsMem.base()) + NumStubs * StubSize; - return reinterpret_cast<void **>(PtrsBase) + Idx; - } - -private: - unsigned NumStubs = 0; - sys::OwningMemoryBlock StubsMem; }; class OrcAArch64 { public: - static const unsigned PointerSize = 8; - static const unsigned TrampolineSize = 12; - static const unsigned ResolverCodeSize = 0x120; - - using IndirectStubsInfo = GenericIndirectStubsInfo<8>; - - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 12; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1U << 27; + static constexpr unsigned ResolverCodeSize = 0x120; /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr); + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress RentryCtxAddr); /// Write the requested number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines); - /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. - /// - /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k - /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 - /// will return a block of 1024 (2-pages worth). - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, void *InitialPtrVal); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned MinStubs); }; /// X86_64 code that's common to all ABIs. @@ -149,25 +128,26 @@ public: /// X86_64 supports lazy JITing. class OrcX86_64_Base { public: - static const unsigned PointerSize = 8; - static const unsigned TrampolineSize = 8; - - using IndirectStubsInfo = GenericIndirectStubsInfo<8>; + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 8; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; /// Write the requested number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines); - /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. - /// - /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k - /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 - /// will return a block of 1024 (2-pages worth). - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, void *InitialPtrVal); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); }; /// X86_64 support for SysV ABI (Linux, MacOSX). @@ -175,15 +155,19 @@ public: /// X86_64_SysV supports lazy JITing. class OrcX86_64_SysV : public OrcX86_64_Base { public: - static const unsigned ResolverCodeSize = 0x6C; - - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); + static constexpr unsigned ResolverCodeSize = 0x6C; /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr); + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); }; /// X86_64 support for Win32. @@ -191,15 +175,19 @@ public: /// X86_64_Win32 supports lazy JITing. class OrcX86_64_Win32 : public OrcX86_64_Base { public: - static const unsigned ResolverCodeSize = 0x74; - - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); + static constexpr unsigned ResolverCodeSize = 0x74; /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr); + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); }; /// I386 support. @@ -207,34 +195,39 @@ public: /// I386 supports lazy JITing. class OrcI386 { public: - static const unsigned PointerSize = 4; - static const unsigned TrampolineSize = 8; - static const unsigned ResolverCodeSize = 0x4a; - - using IndirectStubsInfo = GenericIndirectStubsInfo<8>; - - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); + static constexpr unsigned PointerSize = 4; + static constexpr unsigned TrampolineSize = 8; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0x4a; /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr); + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); /// Write the requested number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines); - /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. - /// - /// E.g. Asking for 4 stubs on i386, where stubs are 8-bytes, with 4k - /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 - /// will return a block of 1024 (2-pages worth). - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, void *InitialPtrVal); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); }; // @brief Mips32 support. @@ -242,41 +235,61 @@ public: // Mips32 supports lazy JITing. class OrcMips32_Base { public: - static const unsigned PointerSize = 4; - static const unsigned TrampolineSize = 20; - static const unsigned ResolverCodeSize = 0xfc; - using IndirectStubsInfo = GenericIndirectStubsInfo<16>; + static constexpr unsigned PointerSize = 4; + static constexpr unsigned TrampolineSize = 20; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0xfc; - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); /// Write the requested number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,unsigned NumTrampolines); + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines); /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr, bool isBigEndian); - /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. /// - /// E.g. Asking for 4 stubs on Mips32, where stubs are 8-bytes, with 4k - /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 - /// will return a block of 1024 (2-pages worth). - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,unsigned MinStubs, void *InitialPtrVal); + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverBlockWorkingMem, + JITTargetAddress ResolverBlockTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr, + bool isBigEndian); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); }; - class OrcMips32Le : public OrcMips32_Base { public: - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr) - { OrcMips32_Base::writeResolverCode(ResolveMem, Reentry, CallbackMgr, false); } + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + OrcMips32_Base::writeResolverCode(ResolverWorkingMem, ResolverTargetAddress, + ReentryFnAddr, ReentryCtxAddr, false); + } }; class OrcMips32Be : public OrcMips32_Base { public: - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr) - { OrcMips32_Base::writeResolverCode(ResolveMem, Reentry, CallbackMgr, true); } + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + OrcMips32_Base::writeResolverCode(ResolverWorkingMem, ResolverTargetAddress, + ReentryFnAddr, ReentryCtxAddr, true); + } }; // @brief Mips64 support. @@ -284,31 +297,41 @@ public: // Mips64 supports lazy JITing. class OrcMips64 { public: - static const unsigned PointerSize = 8; - static const unsigned TrampolineSize = 40; - static const unsigned ResolverCodeSize = 0x120; + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 40; + static constexpr unsigned StubSize = 32; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0x120; - using IndirectStubsInfo = GenericIndirectStubsInfo<32>; - using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, - void *TrampolineId); /// Write the resolver code into the given memory. The user is /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr); + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); /// Write the requested number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,unsigned NumTrampolines); - - /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. - /// - /// E.g. Asking for 4 stubs on Mips64, where stubs are 8-bytes, with 4k - /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 - /// will return a block of 1024 (2-pages worth). - static Error emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,unsigned MinStubs, void *InitialPtrVal); + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverFnAddr, + unsigned NumTrampolines); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); }; - } // end namespace orc - } // end namespace llvm +} // end namespace orc +} // end namespace llvm + #endif // LLVM_EXECUTIONENGINE_ORC_ORCABISUPPORT_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h index 61e2e49a872a..9b0d941f5459 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -37,7 +37,9 @@ enum class OrcErrorCode : int { UnexpectedRPCCall, UnexpectedRPCResponse, UnknownErrorCodeFromRemote, - UnknownResourceHandle + UnknownResourceHandle, + MissingSymbolDefinitions, + UnexpectedSymbolDefinitions, }; std::error_code orcError(OrcErrorCode ErrCode); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h index 3ff5a5f6e90e..52a328165240 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -108,8 +108,7 @@ public: template <typename ChannelT> class SerializationTraits< ChannelT, remote::DirectBufferWriter, remote::DirectBufferWriter, - typename std::enable_if< - std::is_base_of<RawByteChannel, ChannelT>::value>::type> { + std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> { public: static Error serialize(ChannelT &C, const remote::DirectBufferWriter &DBW) { if (auto EC = serializeSeq(C, DBW.getDst())) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h index 4c8e2ea1a7be..50c155d77db1 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -15,6 +15,7 @@ #define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETSERVER_H #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" #include "llvm/Support/Debug.h" @@ -60,7 +61,7 @@ public: SymbolLookup(std::move(SymbolLookup)), EHFramesRegister(std::move(EHFramesRegister)), EHFramesDeregister(std::move(EHFramesDeregister)) { - using ThisT = typename std::remove_reference<decltype(*this)>::type; + using ThisT = std::remove_reference_t<decltype(*this)>; addHandler<exec::CallIntVoid>(*this, &ThisT::handleCallIntVoid); addHandler<exec::CallMain>(*this, &ThisT::handleCallMain); addHandler<exec::CallVoidVoid>(*this, &ThisT::handleCallVoidVoid); @@ -262,19 +263,17 @@ private: return errorCodeToError( orcError(OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist)); - typename TargetT::IndirectStubsInfo IS; - if (auto Err = - TargetT::emitIndirectStubsBlock(IS, NumStubsRequired, nullptr)) - return std::move(Err); + auto IS = LocalIndirectStubsInfo<TargetT>::create( + NumStubsRequired, sys::Process::getPageSizeEstimate()); + if (!IS) + return IS.takeError(); - JITTargetAddress StubsBase = static_cast<JITTargetAddress>( - reinterpret_cast<uintptr_t>(IS.getStub(0))); - JITTargetAddress PtrsBase = static_cast<JITTargetAddress>( - reinterpret_cast<uintptr_t>(IS.getPtr(0))); - uint32_t NumStubsEmitted = IS.getNumStubs(); + JITTargetAddress StubsBase = pointerToJITTargetAddress(IS->getStub(0)); + JITTargetAddress PtrsBase = pointerToJITTargetAddress(IS->getPtr(0)); + uint32_t NumStubsEmitted = IS->getNumStubs(); auto &BlockList = StubOwnerItr->second; - BlockList.push_back(std::move(IS)); + BlockList.push_back(std::move(*IS)); return std::make_tuple(StubsBase, PtrsBase, NumStubsEmitted); } @@ -287,8 +286,10 @@ private: if (EC) return errorCodeToError(EC); - TargetT::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()), - &reenter, this); + TargetT::writeResolverCode(static_cast<char *>(ResolverBlock.base()), + pointerToJITTargetAddress(ResolverBlock.base()), + pointerToJITTargetAddress(&reenter), + pointerToJITTargetAddress(this)); return errorCodeToError(sys::Memory::protectMappedMemory( ResolverBlock.getMemoryBlock(), @@ -308,9 +309,10 @@ private: (sys::Process::getPageSizeEstimate() - TargetT::PointerSize) / TargetT::TrampolineSize; - uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base()); - TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), - NumTrampolines); + char *TrampolineMem = static_cast<char *>(TrampolineBlock.base()); + TargetT::writeTrampolines( + TrampolineMem, pointerToJITTargetAddress(TrampolineMem), + pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines); EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), sys::Memory::MF_READ | @@ -318,10 +320,8 @@ private: TrampolineBlocks.push_back(std::move(TrampolineBlock)); - auto TrampolineBaseAddr = static_cast<JITTargetAddress>( - reinterpret_cast<uintptr_t>(TrampolineMem)); - - return std::make_tuple(TrampolineBaseAddr, NumTrampolines); + return std::make_tuple(pointerToJITTargetAddress(TrampolineMem), + NumTrampolines); } Expected<JITTargetAddress> handleGetSymbolAddress(const std::string &Name) { @@ -337,7 +337,7 @@ private: uint32_t PointerSize = TargetT::PointerSize; uint32_t PageSize = sys::Process::getPageSizeEstimate(); uint32_t TrampolineSize = TargetT::TrampolineSize; - uint32_t IndirectStubSize = TargetT::IndirectStubsInfo::StubSize; + uint32_t IndirectStubSize = TargetT::StubSize; LLVM_DEBUG(dbgs() << " Remote info:\n" << " triple = '" << ProcessTriple << "'\n" << " pointer size = " << PointerSize << "\n" @@ -433,7 +433,7 @@ private: SymbolLookupFtor SymbolLookup; EHFrameRegistrationFtor EHFramesRegister, EHFramesDeregister; std::map<ResourceIdMgr::ResourceId, Allocator> Allocators; - using ISBlockOwnerList = std::vector<typename TargetT::IndirectStubsInfo>; + using ISBlockOwnerList = std::vector<LocalIndirectStubsInfo<TargetT>>; std::map<ResourceIdMgr::ResourceId, ISBlockOwnerList> IndirectStubsOwners; sys::OwningMemoryBlock ResolverBlock; std::vector<sys::OwningMemoryBlock> TrampolineBlocks; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h index 9c69a84f4c67..2f37ab40c7f8 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h @@ -230,9 +230,9 @@ public: /// /// template <DerivedChannelT> /// class SerializationTraits<DerivedChannelT, bool, -/// typename std::enable_if< +/// std::enable_if_t< /// std::is_base_of<VirtChannel, DerivedChannel>::value -/// >::type> { +/// >> { /// public: /// static const char* getName() { ... }; /// } @@ -274,9 +274,8 @@ public: template <typename CArgT> static Error serialize(ChannelT &C, CArgT &&CArg) { - return SerializationTraits<ChannelT, ArgT, - typename std::decay<CArgT>::type>:: - serialize(C, std::forward<CArgT>(CArg)); + return SerializationTraits<ChannelT, ArgT, std::decay_t<CArgT>>::serialize( + C, std::forward<CArgT>(CArg)); } template <typename CArgT> @@ -293,8 +292,8 @@ public: static Error serialize(ChannelT &C, CArgT &&CArg, CArgTs &&... CArgs) { if (auto Err = - SerializationTraits<ChannelT, ArgT, typename std::decay<CArgT>::type>:: - serialize(C, std::forward<CArgT>(CArg))) + SerializationTraits<ChannelT, ArgT, std::decay_t<CArgT>>::serialize( + C, std::forward<CArgT>(CArg))) return Err; if (auto Err = SequenceTraits<ChannelT>::emitSeparator(C)) return Err; @@ -316,8 +315,8 @@ public: template <typename ChannelT, typename... ArgTs> Error serializeSeq(ChannelT &C, ArgTs &&... Args) { - return SequenceSerialization<ChannelT, typename std::decay<ArgTs>::type...>:: - serialize(C, std::forward<ArgTs>(Args)...); + return SequenceSerialization<ChannelT, std::decay_t<ArgTs>...>::serialize( + C, std::forward<ArgTs>(Args)...); } template <typename ChannelT, typename... ArgTs> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h index ed09363dcecc..f348844f39ce 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h @@ -184,8 +184,7 @@ template <typename T, typename = void> class RPCFunctionIdAllocator; /// This specialization of RPCFunctionIdAllocator provides a default /// implementation for integral types. template <typename T> -class RPCFunctionIdAllocator< - T, typename std::enable_if<std::is_integral<T>::value>::type> { +class RPCFunctionIdAllocator<T, std::enable_if_t<std::is_integral<T>::value>> { public: static T getInvalidId() { return T(0); } static T getResponseId() { return T(1); } @@ -205,8 +204,7 @@ template <typename T> class FunctionArgsTuple; template <typename RetT, typename... ArgTs> class FunctionArgsTuple<RetT(ArgTs...)> { public: - using Type = std::tuple<typename std::decay< - typename std::remove_reference<ArgTs>::type>::type...>; + using Type = std::tuple<std::decay_t<std::remove_reference_t<ArgTs>>...>; }; // ResultTraits provides typedefs and utilities specific to the return type @@ -483,9 +481,9 @@ public: }; template <typename ResponseHandlerT, typename... ArgTs> -class AsyncHandlerTraits<Error(ResponseHandlerT, ArgTs...)> : - public AsyncHandlerTraits<Error(typename std::decay<ResponseHandlerT>::type, - ArgTs...)> {}; +class AsyncHandlerTraits<Error(ResponseHandlerT, ArgTs...)> + : public AsyncHandlerTraits<Error(std::decay_t<ResponseHandlerT>, + ArgTs...)> {}; // This template class provides utilities related to RPC function handlers. // The base case applies to non-function types (the template class is @@ -524,18 +522,17 @@ public: // Call the given handler with the given arguments. template <typename HandlerT> - static typename std::enable_if< - std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value, - Error>::type + static std::enable_if_t< + std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value, Error> run(HandlerT &Handler, ArgTs &&... Args) { Handler(std::move(Args)...); return Error::success(); } template <typename HandlerT, typename... TArgTs> - static typename std::enable_if< + static std::enable_if_t< !std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value, - typename HandlerTraits<HandlerT>::ReturnType>::type + typename HandlerTraits<HandlerT>::ReturnType> run(HandlerT &Handler, TArgTs... Args) { return Handler(std::move(Args)...); } @@ -894,12 +891,12 @@ private: using S = SerializationTraits<ChannelT, WireT, ConcreteT>; template <typename T> - static std::true_type - check(typename std::enable_if< - std::is_same<decltype(T::serialize(std::declval<ChannelT &>(), - std::declval<const ConcreteT &>())), - Error>::value, - void *>::type); + static std::true_type check( + std::enable_if_t<std::is_same<decltype(T::serialize( + std::declval<ChannelT &>(), + std::declval<const ConcreteT &>())), + Error>::value, + void *>); template <typename> static std::false_type check(...); @@ -914,11 +911,11 @@ private: template <typename T> static std::true_type - check(typename std::enable_if< - std::is_same<decltype(T::deserialize(std::declval<ChannelT &>(), - std::declval<ConcreteT &>())), - Error>::value, - void *>::type); + check(std::enable_if_t< + std::is_same<decltype(T::deserialize(std::declval<ChannelT &>(), + std::declval<ConcreteT &>())), + Error>::value, + void *>); template <typename> static std::false_type check(...); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h index 50e26f8449df..35745993248c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h @@ -87,13 +87,13 @@ private: template <typename ChannelT, typename T> class SerializationTraits< ChannelT, T, T, - typename std::enable_if< + std::enable_if_t< std::is_base_of<RawByteChannel, ChannelT>::value && (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value || std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value || std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value || std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value || - std::is_same<T, char>::value)>::type> { + std::is_same<T, char>::value)>> { public: static Error serialize(ChannelT &C, T V) { support::endian::byte_swap<T, support::big>(V); @@ -109,9 +109,9 @@ public: }; template <typename ChannelT> -class SerializationTraits<ChannelT, bool, bool, - typename std::enable_if<std::is_base_of< - RawByteChannel, ChannelT>::value>::type> { +class SerializationTraits< + ChannelT, bool, bool, + std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> { public: static Error serialize(ChannelT &C, bool V) { uint8_t Tmp = V ? 1 : 0; @@ -131,9 +131,9 @@ public: }; template <typename ChannelT> -class SerializationTraits<ChannelT, std::string, StringRef, - typename std::enable_if<std::is_base_of< - RawByteChannel, ChannelT>::value>::type> { +class SerializationTraits< + ChannelT, std::string, StringRef, + std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> { public: /// RPC channel serialization for std::strings. static Error serialize(RawByteChannel &C, StringRef S) { @@ -144,11 +144,11 @@ public: }; template <typename ChannelT, typename T> -class SerializationTraits<ChannelT, std::string, T, - typename std::enable_if< - std::is_base_of<RawByteChannel, ChannelT>::value && - (std::is_same<T, const char*>::value || - std::is_same<T, char*>::value)>::type> { +class SerializationTraits< + ChannelT, std::string, T, + std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value && + (std::is_same<T, const char *>::value || + std::is_same<T, char *>::value)>> { public: static Error serialize(RawByteChannel &C, const char *S) { return SerializationTraits<ChannelT, std::string, StringRef>::serialize(C, @@ -157,9 +157,9 @@ public: }; template <typename ChannelT> -class SerializationTraits<ChannelT, std::string, std::string, - typename std::enable_if<std::is_base_of< - RawByteChannel, ChannelT>::value>::type> { +class SerializationTraits< + ChannelT, std::string, std::string, + std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> { public: /// RPC channel serialization for std::strings. static Error serialize(RawByteChannel &C, const std::string &S) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 091394795c0b..9ada0871cf0c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -16,6 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/Layer.h" @@ -115,15 +116,23 @@ public: return *this; } + /// Register a JITEventListener. + void registerJITEventListener(JITEventListener &L); + + /// Unregister a JITEventListener. + void unregisterJITEventListener(JITEventListener &L); + private: Error onObjLoad(VModuleKey K, MaterializationResponsibility &R, - object::ObjectFile &Obj, + const object::ObjectFile &Obj, + RuntimeDyld::MemoryManager *MemMgr, std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, std::map<StringRef, JITEvaluatedSymbol> Resolved, std::set<StringRef> &InternalSymbols); - void onObjEmit(VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer, - MaterializationResponsibility &R, Error Err); + void onObjEmit(VModuleKey K, MaterializationResponsibility &R, + object::OwningBinary<object::ObjectFile> O, + RuntimeDyld::MemoryManager *MemMgr, Error Err); mutable std::mutex RTDyldLayerMutex; GetMemoryManagerFunction GetMemoryManager; @@ -133,6 +142,10 @@ private: bool OverrideObjectFlags = false; bool AutoClaimObjectSymbols = false; std::vector<std::unique_ptr<RuntimeDyld::MemoryManager>> MemMgrs; + std::vector<JITEventListener *> EventListeners; + DenseMap<RuntimeDyld::MemoryManager *, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo>> + LoadedObjInfos; }; class LegacyRTDyldObjectLinkingLayerBase { @@ -170,7 +183,7 @@ protected: if (!SymEntry->second.getFlags().isExported() && ExportedSymbolsOnly) return nullptr; if (!Finalized) - return JITSymbol(getSymbolMaterializer(Name), + return JITSymbol(getSymbolMaterializer(std::string(Name)), SymEntry->second.getFlags()); return JITSymbol(SymEntry->second); } @@ -291,8 +304,15 @@ private: private: void buildInitialSymbolTable(const OwnedObject &Obj) { for (auto &Symbol : Obj.getBinary()->symbols()) { - if (Symbol.getFlags() & object::SymbolRef::SF_Undefined) + if (Expected<uint32_t> SymbolFlagsOrErr = Symbol.getFlags()) { + if (*SymbolFlagsOrErr & object::SymbolRef::SF_Undefined) + continue; + } else { + // FIXME: Raise an error for bad symbols. + consumeError(SymbolFlagsOrErr.takeError()); continue; + } + Expected<StringRef> SymbolName = Symbol.getName(); // FIXME: Raise an error for bad symbols. if (!SymbolName) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h index f6b86bb23167..d8213d3b35e8 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -13,10 +13,10 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATION_H #define LLVM_EXECUTIONENGINE_ORC_SPECULATION_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/IR/PassManager.h" #include "llvm/Passes/PassBuilder.h" @@ -182,8 +182,8 @@ public: IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, Speculator &Spec, MangleAndInterner &Mangle, ResultEval Interpreter) - : IRLayer(ES), NextLayer(BaseLayer), S(Spec), Mangle(Mangle), - QueryAnalysis(Interpreter) {} + : IRLayer(ES, BaseLayer.getManglingOptions()), NextLayer(BaseLayer), + S(Spec), Mangle(Mangle), QueryAnalysis(Interpreter) {} void emit(MaterializationResponsibility R, ThreadSafeModule TSM); @@ -192,11 +192,10 @@ private: internToJITSymbols(DenseMap<StringRef, DenseSet<StringRef>> IRNames) { assert(!IRNames.empty() && "No IRNames received to Intern?"); TargetAndLikelies InternedNames; - DenseSet<SymbolStringPtr> TargetJITNames; for (auto &NamePair : IRNames) { + DenseSet<SymbolStringPtr> TargetJITNames; for (auto &TargetNames : NamePair.second) TargetJITNames.insert(Mangle(TargetNames)); - InternedNames[Mangle(NamePair.first)] = std::move(TargetJITNames); } return InternedNames; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index c354f6c3559c..c9fadd727e88 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -48,11 +48,13 @@ private: /// Pointer to a pooled string representing a symbol name. class SymbolStringPtr { + friend class OrcV2CAPIHelper; friend class SymbolStringPool; friend struct DenseMapInfo<SymbolStringPtr>; public: SymbolStringPtr() = default; + SymbolStringPtr(std::nullptr_t) {} SymbolStringPtr(const SymbolStringPtr &Other) : S(Other.S) { if (isRealPoolEntry(S)) @@ -85,6 +87,8 @@ public: --S->getValue(); } + explicit operator bool() const { return S; } + StringRef operator*() const { return S->first(); } friend bool operator==(const SymbolStringPtr &LHS, @@ -103,7 +107,8 @@ public: } private: - using PoolEntryPtr = SymbolStringPool::PoolMapEntry *; + using PoolEntry = SymbolStringPool::PoolMapEntry; + using PoolEntryPtr = PoolEntry *; SymbolStringPtr(SymbolStringPool::PoolMapEntry *S) : S(S) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h index 2347faed37a2..58c96737e580 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -130,8 +130,7 @@ public: /// Locks the associated ThreadSafeContext and calls the given function /// on the contained Module. - template <typename Func> - auto withModuleDo(Func &&F) -> decltype(F(std::declval<Module &>())) { + template <typename Func> decltype(auto) withModuleDo(Func &&F) { assert(M && "Can not call on null module"); auto Lock = TSCtx.getLock(); return F(*M); @@ -139,9 +138,7 @@ public: /// Locks the associated ThreadSafeContext and calls the given function /// on the contained Module. - template <typename Func> - auto withModuleDo(Func &&F) const - -> decltype(F(std::declval<const Module &>())) { + template <typename Func> decltype(auto) withModuleDo(Func &&F) const { auto Lock = TSCtx.getLock(); return F(*M); } diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h index ce7024a7f19b..1b3ce1127e4a 100644 --- a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -267,15 +267,16 @@ public: void finalizeWithMemoryManagerLocking(); private: - friend void - jitLinkForORC(object::ObjectFile &Obj, - std::unique_ptr<MemoryBuffer> UnderlyingBuffer, - RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, - bool ProcessAllSections, - unique_function<Error(std::unique_ptr<LoadedObjectInfo>, - std::map<StringRef, JITEvaluatedSymbol>)> - OnLoaded, - unique_function<void(Error)> OnEmitted); + friend void jitLinkForORC( + object::OwningBinary<object::ObjectFile> O, + RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, + bool ProcessAllSections, + unique_function<Error(const object::ObjectFile &Obj, + std::unique_ptr<LoadedObjectInfo>, + std::map<StringRef, JITEvaluatedSymbol>)> + OnLoaded, + unique_function<void(object::OwningBinary<object::ObjectFile> O, Error)> + OnEmitted); // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public // interface. @@ -293,13 +294,15 @@ private: // instance and uses continuation passing to perform the fix-up and finalize // steps asynchronously. void jitLinkForORC( - object::ObjectFile &Obj, std::unique_ptr<MemoryBuffer> UnderlyingBuffer, + object::OwningBinary<object::ObjectFile> O, RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, bool ProcessAllSections, - unique_function<Error(std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, + unique_function<Error(const object::ObjectFile &Obj, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, std::map<StringRef, JITEvaluatedSymbol>)> OnLoaded, - unique_function<void(Error)> OnEmitted); + unique_function<void(object::OwningBinary<object::ObjectFile>, Error)> + OnEmitted); } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h index d7316425da2f..49956fab17ce 100644 --- a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -15,7 +15,6 @@ #define LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/Support/Memory.h" #include <cstdint> |