diff options
Diffstat (limited to 'include/llvm/ExecutionEngine/Orc')
23 files changed, 2328 insertions, 789 deletions
diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index a961992c2147..8bd21a0e3dd6 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -22,6 +22,8 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Attributes.h" @@ -56,7 +58,47 @@ class Value; namespace orc { -/// @brief Compile-on-demand layer. +class ExtractingIRMaterializationUnit; + +class CompileOnDemandLayer2 : public IRLayer { + friend class ExtractingIRMaterializationUnit; + +public: + /// Builder for IndirectStubsManagers. + using IndirectStubsManagerBuilder = + std::function<std::unique_ptr<IndirectStubsManager>()>; + + using GetAvailableContextFunction = std::function<LLVMContext &()>; + + CompileOnDemandLayer2(ExecutionSession &ES, IRLayer &BaseLayer, + JITCompileCallbackManager &CCMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager, + GetAvailableContextFunction GetAvailableContext); + + Error add(VSO &V, VModuleKey K, std::unique_ptr<Module> M) override; + + void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) override; + +private: + using StubManagersMap = + std::map<const VSO *, std::unique_ptr<IndirectStubsManager>>; + + IndirectStubsManager &getStubsManager(const VSO &V); + + void emitExtractedFunctionsModule(MaterializationResponsibility R, + std::unique_ptr<Module> M); + + mutable std::mutex CODLayerMutex; + + IRLayer &BaseLayer; + JITCompileCallbackManager &CCMgr; + IndirectStubsManagerBuilder BuildIndirectStubsManager; + StubManagersMap StubsMgrs; + GetAvailableContextFunction GetAvailableContext; +}; + +/// Compile-on-demand layer. /// /// When a module is added to this layer a stub is created for each of its /// function definitions. The stubs and other global values are immediately @@ -85,8 +127,6 @@ private: return LambdaMaterializer<MaterializerFtor>(std::move(M)); } - using BaseLayerModuleHandleT = typename BaseLayerT::ModuleHandleT; - // Provide type-erasure for the Modules and MemoryManagers. template <typename ResourceT> class ResourceOwner { @@ -138,18 +178,22 @@ private: }; struct LogicalDylib { - using SymbolResolverFtor = std::function<JITSymbol(const std::string&)>; - struct SourceModuleEntry { - std::shared_ptr<Module> SourceMod; + std::unique_ptr<Module> SourceMod; std::set<Function*> StubsToClone; }; using SourceModulesList = std::vector<SourceModuleEntry>; using SourceModuleHandle = typename SourceModulesList::size_type; - SourceModuleHandle - addSourceModule(std::shared_ptr<Module> M) { + LogicalDylib() = default; + + LogicalDylib(VModuleKey K, std::shared_ptr<SymbolResolver> BackingResolver, + std::unique_ptr<IndirectStubsMgrT> StubsMgr) + : K(std::move(K)), BackingResolver(std::move(BackingResolver)), + StubsMgr(std::move(StubsMgr)) {} + + SourceModuleHandle addSourceModule(std::unique_ptr<Module> M) { SourceModuleHandle H = SourceModules.size(); SourceModules.push_back(SourceModuleEntry()); SourceModules.back().SourceMod = std::move(M); @@ -168,8 +212,8 @@ private: bool ExportedSymbolsOnly) { if (auto Sym = StubsMgr->findStub(Name, ExportedSymbolsOnly)) return Sym; - for (auto BLH : BaseLayerHandles) - if (auto Sym = BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly)) + for (auto BLK : BaseLayerVModuleKeys) + if (auto Sym = BaseLayer.findSymbolIn(BLK, Name, ExportedSymbolsOnly)) return Sym; else if (auto Err = Sym.takeError()) return std::move(Err); @@ -177,91 +221,94 @@ private: } Error removeModulesFromBaseLayer(BaseLayerT &BaseLayer) { - for (auto &BLH : BaseLayerHandles) - if (auto Err = BaseLayer.removeModule(BLH)) + for (auto &BLK : BaseLayerVModuleKeys) + if (auto Err = BaseLayer.removeModule(BLK)) return Err; return Error::success(); } - std::shared_ptr<JITSymbolResolver> ExternalSymbolResolver; + VModuleKey K; + std::shared_ptr<SymbolResolver> BackingResolver; std::unique_ptr<IndirectStubsMgrT> StubsMgr; StaticGlobalRenamer StaticRenamer; SourceModulesList SourceModules; - std::vector<BaseLayerModuleHandleT> BaseLayerHandles; + std::vector<VModuleKey> BaseLayerVModuleKeys; }; - using LogicalDylibList = std::list<LogicalDylib>; - public: - /// @brief Handle to loaded module. - using ModuleHandleT = typename LogicalDylibList::iterator; - - /// @brief Module partitioning functor. + /// Module partitioning functor. using PartitioningFtor = std::function<std::set<Function*>(Function&)>; - /// @brief Builder for IndirectStubsManagers. + /// Builder for IndirectStubsManagers. using IndirectStubsManagerBuilderT = std::function<std::unique_ptr<IndirectStubsMgrT>()>; - /// @brief Construct a compile-on-demand layer instance. - CompileOnDemandLayer(BaseLayerT &BaseLayer, PartitioningFtor Partition, + using SymbolResolverGetter = + std::function<std::shared_ptr<SymbolResolver>(VModuleKey K)>; + + using SymbolResolverSetter = + std::function<void(VModuleKey K, std::shared_ptr<SymbolResolver> R)>; + + /// Construct a compile-on-demand layer instance. + CompileOnDemandLayer(ExecutionSession &ES, BaseLayerT &BaseLayer, + SymbolResolverGetter GetSymbolResolver, + SymbolResolverSetter SetSymbolResolver, + PartitioningFtor Partition, CompileCallbackMgrT &CallbackMgr, IndirectStubsManagerBuilderT CreateIndirectStubsManager, bool CloneStubsIntoPartitions = true) - : BaseLayer(BaseLayer), Partition(std::move(Partition)), - CompileCallbackMgr(CallbackMgr), + : ES(ES), BaseLayer(BaseLayer), + GetSymbolResolver(std::move(GetSymbolResolver)), + SetSymbolResolver(std::move(SetSymbolResolver)), + Partition(std::move(Partition)), CompileCallbackMgr(CallbackMgr), CreateIndirectStubsManager(std::move(CreateIndirectStubsManager)), CloneStubsIntoPartitions(CloneStubsIntoPartitions) {} ~CompileOnDemandLayer() { // FIXME: Report error on log. while (!LogicalDylibs.empty()) - consumeError(removeModule(LogicalDylibs.begin())); + consumeError(removeModule(LogicalDylibs.begin()->first)); } - /// @brief Add a module to the compile-on-demand layer. - Expected<ModuleHandleT> - addModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) { + /// Add a module to the compile-on-demand layer. + Error addModule(VModuleKey K, std::unique_ptr<Module> M) { - LogicalDylibs.push_back(LogicalDylib()); - auto &LD = LogicalDylibs.back(); - LD.ExternalSymbolResolver = std::move(Resolver); - LD.StubsMgr = CreateIndirectStubsManager(); + assert(!LogicalDylibs.count(K) && "VModuleKey K already in use"); + auto I = LogicalDylibs.insert( + LogicalDylibs.end(), + std::make_pair(K, LogicalDylib(K, GetSymbolResolver(K), + CreateIndirectStubsManager()))); - // Process each of the modules in this module set. - if (auto Err = addLogicalModule(LD, std::move(M))) - return std::move(Err); - - return std::prev(LogicalDylibs.end()); + return addLogicalModule(I->second, std::move(M)); } - /// @brief Add extra modules to an existing logical module. - Error addExtraModule(ModuleHandleT H, std::shared_ptr<Module> M) { - return addLogicalModule(*H, std::move(M)); + /// Add extra modules to an existing logical module. + Error addExtraModule(VModuleKey K, std::unique_ptr<Module> M) { + return addLogicalModule(LogicalDylibs[K], std::move(M)); } - /// @brief Remove the module represented by the given handle. + /// Remove the module represented by the given key. /// /// This will remove all modules in the layers below that were derived from - /// the module represented by H. - Error removeModule(ModuleHandleT H) { - auto Err = H->removeModulesFromBaseLayer(BaseLayer); - LogicalDylibs.erase(H); + /// the module represented by K. + Error removeModule(VModuleKey K) { + auto I = LogicalDylibs.find(K); + assert(I != LogicalDylibs.end() && "VModuleKey K not valid here"); + auto Err = I->second.removeModulesFromBaseLayer(BaseLayer); + LogicalDylibs.erase(I); return Err; } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { - for (auto LDI = LogicalDylibs.begin(), LDE = LogicalDylibs.end(); - LDI != LDE; ++LDI) { - if (auto Sym = LDI->StubsMgr->findStub(Name, ExportedSymbolsOnly)) + for (auto &KV : LogicalDylibs) { + if (auto Sym = KV.second.StubsMgr->findStub(Name, ExportedSymbolsOnly)) return Sym; - if (auto Sym = findSymbolIn(LDI, Name, ExportedSymbolsOnly)) + if (auto Sym = findSymbolIn(KV.first, Name, ExportedSymbolsOnly)) return Sym; else if (auto Err = Sym.takeError()) return std::move(Err); @@ -269,14 +316,15 @@ public: return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of a symbol provided by this layer, or some layer + /// Get the address of a symbol provided by this layer, or some layer /// below this one. - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + JITSymbol findSymbolIn(VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return H->findSymbol(BaseLayer, Name, ExportedSymbolsOnly); + assert(LogicalDylibs.count(K) && "VModuleKey K is not valid here"); + return LogicalDylibs[K].findSymbol(BaseLayer, Name, ExportedSymbolsOnly); } - /// @brief Update the stub for the given function to point at FnBodyAddr. + /// Update the stub for the given function to point at FnBodyAddr. /// This can be used to support re-optimization. /// @return true if the function exists and the stub is updated, false /// otherwise. @@ -302,15 +350,14 @@ public: } private: - - Error addLogicalModule(LogicalDylib &LD, std::shared_ptr<Module> SrcMPtr) { + Error addLogicalModule(LogicalDylib &LD, std::unique_ptr<Module> SrcMPtr) { // Rename all static functions / globals to $static.X : // This will unique the names across all modules in the logical dylib, // simplifying symbol lookup. LD.StaticRenamer.rename(*SrcMPtr); - // Bump the linkage and rename any anonymous/privote members in SrcM to + // Bump the linkage and rename any anonymous/private members in SrcM to // ensure that everything will resolve properly after we partition SrcM. makeAllSymbolsExternallyAccessible(*SrcMPtr); @@ -343,22 +390,21 @@ private: // Create a callback, associate it with the stub for the function, // and set the compile action to compile the partition containing the // function. - if (auto CCInfoOrErr = CompileCallbackMgr.getCompileCallback()) { - auto &CCInfo = *CCInfoOrErr; + auto CompileAction = [this, &LD, LMId, &F]() -> JITTargetAddress { + if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) + return *FnImplAddrOrErr; + else { + // FIXME: Report error, return to 'abort' or something similar. + consumeError(FnImplAddrOrErr.takeError()); + return 0; + } + }; + if (auto CCAddr = + CompileCallbackMgr.getCompileCallback(std::move(CompileAction))) StubInits[MangledName] = - std::make_pair(CCInfo.getAddress(), - JITSymbolFlags::fromGlobalValue(F)); - CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress { - if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) - return *FnImplAddrOrErr; - else { - // FIXME: Report error, return to 'abort' or something similar. - consumeError(FnImplAddrOrErr.takeError()); - return 0; - } - }); - } else - return CCInfoOrErr.takeError(); + std::make_pair(*CCAddr, JITSymbolFlags::fromGlobalValue(F)); + else + return CCAddr.takeError(); } if (auto Err = LD.StubsMgr->createStubs(StubInits)) @@ -396,9 +442,8 @@ private: // Initializers may refer to functions declared (but not defined) in this // module. Build a materializer to clone decls on demand. - Error MaterializerErrors = Error::success(); auto Materializer = createLambdaMaterializer( - [&LD, &GVsM, &MaterializerErrors](Value *V) -> Value* { + [&LD, &GVsM](Value *V) -> Value* { if (auto *F = dyn_cast<Function>(V)) { // Decls in the original module just get cloned. if (F->isDeclaration()) @@ -410,18 +455,8 @@ private: const DataLayout &DL = GVsM->getDataLayout(); std::string FName = mangle(F->getName(), DL); unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType()); - JITTargetAddress StubAddr = 0; - - // Get the address for the stub. If we encounter an error while - // doing so, stash it in the MaterializerErrors variable and use a - // null address as a placeholder. - if (auto StubSym = LD.StubsMgr->findStub(FName, false)) { - if (auto StubAddrOrErr = StubSym.getAddress()) - StubAddr = *StubAddrOrErr; - else - MaterializerErrors = joinErrors(std::move(MaterializerErrors), - StubAddrOrErr.takeError()); - } + JITTargetAddress StubAddr = + LD.StubsMgr->findStub(FName, false).getAddress(); ConstantInt *StubAddrCI = ConstantInt::get(GVsM->getContext(), APInt(PtrBitWidth, StubAddr)); @@ -450,29 +485,58 @@ private: NewA->setAliasee(cast<Constant>(Init)); } - if (MaterializerErrors) - return MaterializerErrors; - // Build a resolver for the globals module and add it to the base layer. - auto GVsResolver = createLambdaResolver( - [this, &LD](const std::string &Name) -> JITSymbol { - if (auto Sym = LD.StubsMgr->findStub(Name, false)) - return Sym; - if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) - return Sym; - else if (auto Err = Sym.takeError()) - return std::move(Err); - return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name); + auto LegacyLookup = [this, &LD](const std::string &Name) -> JITSymbol { + if (auto Sym = LD.StubsMgr->findStub(Name, false)) + return Sym; + + if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) + return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); + + return nullptr; + }; + + auto GVsResolver = createSymbolResolver( + [&LD, LegacyLookup](const SymbolNameSet &Symbols) { + auto SymbolFlags = lookupFlagsWithLegacyFn(Symbols, LegacyLookup); + + if (!SymbolFlags) { + logAllUnhandledErrors(SymbolFlags.takeError(), errs(), + "CODLayer/GVsResolver flags lookup failed: "); + return SymbolFlagsMap(); + } + + if (SymbolFlags->size() == Symbols.size()) + return *SymbolFlags; + + SymbolNameSet NotFoundViaLegacyLookup; + for (auto &S : Symbols) + if (!SymbolFlags->count(S)) + NotFoundViaLegacyLookup.insert(S); + auto SymbolFlags2 = + LD.BackingResolver->lookupFlags(NotFoundViaLegacyLookup); + + for (auto &KV : SymbolFlags2) + (*SymbolFlags)[KV.first] = std::move(KV.second); + + return *SymbolFlags; }, - [&LD](const std::string &Name) { - return LD.ExternalSymbolResolver->findSymbol(Name); + [this, &LD, + LegacyLookup](std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) { + auto NotFoundViaLegacyLookup = + lookupWithLegacyFn(ES, *Query, Symbols, LegacyLookup); + return LD.BackingResolver->lookup(Query, NotFoundViaLegacyLookup); }); - if (auto GVsHOrErr = - BaseLayer.addModule(std::move(GVsM), std::move(GVsResolver))) - LD.BaseLayerHandles.push_back(*GVsHOrErr); - else - return GVsHOrErr.takeError(); + SetSymbolResolver(LD.K, std::move(GVsResolver)); + + if (auto Err = BaseLayer.addModule(LD.K, std::move(GVsM))) + return Err; + + LD.BaseLayerVModuleKeys.push_back(LD.K); return Error::success(); } @@ -501,11 +565,11 @@ private: JITTargetAddress CalledAddr = 0; auto Part = Partition(F); - if (auto PartHOrErr = emitPartition(LD, LMId, Part)) { - auto &PartH = *PartHOrErr; + if (auto PartKeyOrErr = emitPartition(LD, LMId, Part)) { + auto &PartKey = *PartKeyOrErr; for (auto *SubF : Part) { std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout()); - if (auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false)) { + if (auto FnBodySym = BaseLayer.findSymbolIn(PartKey, FnName, false)) { if (auto FnBodyAddrOrErr = FnBodySym.getAddress()) { JITTargetAddress FnBodyAddr = *FnBodyAddrOrErr; @@ -526,15 +590,15 @@ private: llvm_unreachable("Function not emitted for partition"); } - LD.BaseLayerHandles.push_back(PartH); + LD.BaseLayerVModuleKeys.push_back(PartKey); } else - return PartHOrErr.takeError(); + return PartKeyOrErr.takeError(); return CalledAddr; } template <typename PartitionT> - Expected<BaseLayerModuleHandleT> + Expected<VModuleKey> emitPartition(LogicalDylib &LD, typename LogicalDylib::SourceModuleHandle LMId, const PartitionT &Part) { @@ -596,28 +660,62 @@ private: for (auto *F : Part) moveFunctionBody(*F, VMap, &Materializer); + auto K = ES.allocateVModule(); + + auto LegacyLookup = [this, &LD](const std::string &Name) -> JITSymbol { + return LD.findSymbol(BaseLayer, Name, false); + }; + // Create memory manager and symbol resolver. - auto Resolver = createLambdaResolver( - [this, &LD](const std::string &Name) -> JITSymbol { - if (auto Sym = LD.findSymbol(BaseLayer, Name, false)) - return Sym; - else if (auto Err = Sym.takeError()) - return std::move(Err); - return LD.ExternalSymbolResolver->findSymbolInLogicalDylib(Name); + auto Resolver = createSymbolResolver( + [&LD, LegacyLookup](const SymbolNameSet &Symbols) { + auto SymbolFlags = lookupFlagsWithLegacyFn(Symbols, LegacyLookup); + if (!SymbolFlags) { + logAllUnhandledErrors(SymbolFlags.takeError(), errs(), + "CODLayer/SubResolver flags lookup failed: "); + return SymbolFlagsMap(); + } + + if (SymbolFlags->size() == Symbols.size()) + return *SymbolFlags; + + SymbolNameSet NotFoundViaLegacyLookup; + for (auto &S : Symbols) + if (!SymbolFlags->count(S)) + NotFoundViaLegacyLookup.insert(S); + + auto SymbolFlags2 = + LD.BackingResolver->lookupFlags(NotFoundViaLegacyLookup); + + for (auto &KV : SymbolFlags2) + (*SymbolFlags)[KV.first] = std::move(KV.second); + + return *SymbolFlags; }, - [&LD](const std::string &Name) { - return LD.ExternalSymbolResolver->findSymbol(Name); + [this, &LD, LegacyLookup](std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Symbols) { + auto NotFoundViaLegacyLookup = + lookupWithLegacyFn(ES, *Q, Symbols, LegacyLookup); + return LD.BackingResolver->lookup(Q, + std::move(NotFoundViaLegacyLookup)); }); + SetSymbolResolver(K, std::move(Resolver)); + + if (auto Err = BaseLayer.addModule(std::move(K), std::move(M))) + return std::move(Err); - return BaseLayer.addModule(std::move(M), std::move(Resolver)); + return K; } + ExecutionSession &ES; BaseLayerT &BaseLayer; + SymbolResolverGetter GetSymbolResolver; + SymbolResolverSetter SetSymbolResolver; PartitioningFtor Partition; CompileCallbackMgrT &CompileCallbackMgr; IndirectStubsManagerBuilderT CreateIndirectStubsManager; - LogicalDylibList LogicalDylibs; + std::map<VModuleKey, LogicalDylib> LogicalDylibs; bool CloneStubsIntoPartitions; }; diff --git a/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/include/llvm/ExecutionEngine/Orc/CompileUtils.h index b9f7d6accc30..213a59124c85 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -16,13 +16,14 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/ObjectCache.h" -#include "llvm/ExecutionEngine/ObjectMemoryBuffer.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include <algorithm> @@ -35,45 +36,51 @@ class Module; namespace orc { -/// @brief Simple compile functor: Takes a single IR module and returns an -/// ObjectFile. +/// Simple compile functor: Takes a single IR module and returns an ObjectFile. +/// This compiler supports a single compilation thread and LLVMContext only. +/// For multithreaded compilation, use MultiThreadedSimpleCompiler below. class SimpleCompiler { public: + using CompileResult = std::unique_ptr<MemoryBuffer>; - using CompileResult = object::OwningBinary<object::ObjectFile>; - - /// @brief Construct a simple compile functor with the given target. + /// Construct a simple compile functor with the given target. SimpleCompiler(TargetMachine &TM, ObjectCache *ObjCache = nullptr) : TM(TM), ObjCache(ObjCache) {} - /// @brief Set an ObjectCache to query before compiling. + /// Set an ObjectCache to query before compiling. void setObjectCache(ObjectCache *NewCache) { ObjCache = NewCache; } - /// @brief Compile a Module to an ObjectFile. + /// Compile a Module to an ObjectFile. CompileResult operator()(Module &M) { CompileResult CachedObject = tryToLoadFromObjectCache(M); - if (CachedObject.getBinary()) + if (CachedObject) return CachedObject; SmallVector<char, 0> ObjBufferSV; - raw_svector_ostream ObjStream(ObjBufferSV); - - legacy::PassManager PM; - MCContext *Ctx; - if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) - llvm_unreachable("Target does not support MC emission."); - PM.run(M); - std::unique_ptr<MemoryBuffer> ObjBuffer( - new ObjectMemoryBuffer(std::move(ObjBufferSV))); - Expected<std::unique_ptr<object::ObjectFile>> Obj = + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + PM.run(M); + } + + auto ObjBuffer = + llvm::make_unique<SmallVectorMemoryBuffer>(std::move(ObjBufferSV)); + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + if (Obj) { notifyObjectCompiled(M, *ObjBuffer); - return CompileResult(std::move(*Obj), std::move(ObjBuffer)); + return std::move(ObjBuffer); } + // TODO: Actually report errors helpfully. consumeError(Obj.takeError()); - return CompileResult(nullptr, nullptr); + return nullptr; } private: @@ -82,19 +89,7 @@ private: if (!ObjCache) return CompileResult(); - std::unique_ptr<MemoryBuffer> ObjBuffer = ObjCache->getObject(&M); - if (!ObjBuffer) - return CompileResult(); - - Expected<std::unique_ptr<object::ObjectFile>> Obj = - object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); - if (!Obj) { - // TODO: Actually report errors helpfully. - consumeError(Obj.takeError()); - return CompileResult(); - } - - return CompileResult(std::move(*Obj), std::move(ObjBuffer)); + return ObjCache->getObject(&M); } void notifyObjectCompiled(const Module &M, const MemoryBuffer &ObjBuffer) { @@ -106,6 +101,29 @@ private: ObjectCache *ObjCache = nullptr; }; +/// A thread-safe version of SimpleCompiler. +/// +/// This class creates a new TargetMachine and SimpleCompiler instance for each +/// compile. +class MultiThreadedSimpleCompiler { +public: + MultiThreadedSimpleCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache = nullptr) + : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + + void setObjectCache(ObjectCache *ObjCache) { this->ObjCache = ObjCache; } + + std::unique_ptr<MemoryBuffer> operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); + } + +private: + JITTargetMachineBuilder JTMB; + ObjectCache *ObjCache = nullptr; +}; + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h new file mode 100644 index 000000000000..fd03687cfc21 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -0,0 +1,779 @@ +//===------ Core.h -- Core ORC APIs (Layer, JITDylib, etc.) -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Contains core ORC APIs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_CORE_H +#define LLVM_EXECUTIONENGINE_ORC_CORE_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" +#include "llvm/IR/Module.h" + +#include <list> +#include <map> +#include <memory> +#include <set> +#include <vector> + +namespace llvm { +namespace orc { + +// Forward declare some classes. +class AsynchronousSymbolQuery; +class ExecutionSession; +class MaterializationUnit; +class MaterializationResponsibility; +class VSO; + +/// VModuleKey provides a unique identifier (allocated and managed by +/// ExecutionSessions) for a module added to the JIT. +using VModuleKey = uint64_t; + +/// A set of symbol names (represented by SymbolStringPtrs for +// efficiency). +using SymbolNameSet = std::set<SymbolStringPtr>; + +/// Render a SymbolNameSet to an ostream. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols); + +/// A map from symbol names (as SymbolStringPtrs) to JITSymbols +/// (address/flags pairs). +using SymbolMap = std::map<SymbolStringPtr, JITEvaluatedSymbol>; + +/// Render a SymbolMap to an ostream. +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols); + +/// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. +using SymbolFlagsMap = std::map<SymbolStringPtr, JITSymbolFlags>; + +/// Render a SymbolMap to an ostream. +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &Symbols); + +/// A base class for materialization failures that allows the failing +/// symbols to be obtained for logging. +using SymbolDependenceMap = std::map<VSO *, SymbolNameSet>; + +/// Render a SymbolDependendeMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps); + +/// A list of VSO pointers. +using VSOList = std::vector<VSO *>; + +/// Render a VSOList. +raw_ostream &operator<<(raw_ostream &OS, const VSOList &VSOs); + +/// Callback to notify client that symbols have been resolved. +using SymbolsResolvedCallback = std::function<void(Expected<SymbolMap>)>; + +/// Callback to notify client that symbols are ready for execution. +using SymbolsReadyCallback = std::function<void(Error)>; + +/// Callback to register the dependencies for a given query. +using RegisterDependenciesFunction = + std::function<void(const SymbolDependenceMap &)>; + +/// This can be used as the value for a RegisterDependenciesFunction if there +/// are no dependants to register with. +extern RegisterDependenciesFunction NoDependenciesToRegister; + +/// Used to notify a VSO that the given set of symbols failed to materialize. +class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { +public: + static char ID; + + FailedToMaterialize(SymbolNameSet Symbols); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const SymbolNameSet &getSymbols() const { return Symbols; } + +private: + SymbolNameSet Symbols; +}; + +/// Used to notify clients when symbols can not be found during a lookup. +class SymbolsNotFound : public ErrorInfo<SymbolsNotFound> { +public: + static char ID; + + SymbolsNotFound(SymbolNameSet Symbols); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const SymbolNameSet &getSymbols() const { return Symbols; } + +private: + SymbolNameSet Symbols; +}; + +/// Tracks responsibility for materialization, and mediates interactions between +/// MaterializationUnits and VSOs. +/// +/// An instance of this class is passed to MaterializationUnits when their +/// materialize method is called. It allows MaterializationUnits to resolve and +/// finalize symbols, or abandon materialization by notifying any unmaterialized +/// symbols of an error. +class MaterializationResponsibility { + friend class MaterializationUnit; +public: + MaterializationResponsibility(MaterializationResponsibility &&) = default; + MaterializationResponsibility & + operator=(MaterializationResponsibility &&) = default; + + /// Destruct a MaterializationResponsibility instance. In debug mode + /// this asserts that all symbols being tracked have been either + /// finalized or notified of an error. + ~MaterializationResponsibility(); + + /// Returns the target VSO that these symbols are being materialized + /// into. + VSO &getTargetVSO() const { return V; } + + /// Returns the symbol flags map for this responsibility instance. + SymbolFlagsMap getSymbols() { return SymbolFlags; } + + /// Returns the names of any symbols covered by this + /// MaterializationResponsibility object that have queries pending. This + /// information can be used to return responsibility for unrequested symbols + /// back to the VSO via the delegate method. + SymbolNameSet getRequestedSymbols(); + + /// Resolves the given symbols. Individual calls to this method may + /// resolve a subset of the symbols, but all symbols must have been + /// resolved prior to calling finalize. + void resolve(const SymbolMap &Symbols); + + /// Finalizes all symbols tracked by this instance. + void finalize(); + + /// Adds new symbols to the VSO and this responsibility instance. + /// VSO entries start out in the materializing state. + /// + /// This method can be used by materialization units that want to add + /// additional symbols at materialization time (e.g. stubs, compile + /// callbacks, metadata). + Error defineMaterializing(const SymbolFlagsMap &SymbolFlags); + + /// Notify all unfinalized symbols that an error has occurred. + /// This will remove all symbols covered by this MaterializationResponsibilty + /// from V, and send an error to any queries waiting on these symbols. + void failMaterialization(); + + /// Transfers responsibility to the given MaterializationUnit for all + /// symbols defined by that MaterializationUnit. This allows + /// materializers to break up work based on run-time information (e.g. + /// by introspecting which symbols have actually been looked up and + /// materializing only those). + void replace(std::unique_ptr<MaterializationUnit> MU); + + /// Delegates responsibility for the given symbols to the returned + /// materialization responsibility. Useful for breaking up work between + /// threads, or different kinds of materialization processes. + MaterializationResponsibility delegate(const SymbolNameSet &Symbols); + + void addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies); + + /// Add dependencies that apply to all symbols covered by this instance. + void addDependenciesForAll(const SymbolDependenceMap &Dependencies); + +private: + /// Create a MaterializationResponsibility for the given VSO and + /// initial symbols. + MaterializationResponsibility(VSO &V, SymbolFlagsMap SymbolFlags); + + VSO &V; + SymbolFlagsMap SymbolFlags; +}; + +/// A MaterializationUnit represents a set of symbol definitions that can +/// be materialized as a group, or individually discarded (when +/// overriding definitions are encountered). +/// +/// MaterializationUnits are used when providing lazy definitions of symbols to +/// VSOs. The VSO will call materialize when the address of a symbol is +/// requested via the lookup method. The VSO will call discard if a stronger +/// definition is added or already present. +class MaterializationUnit { +public: + MaterializationUnit(SymbolFlagsMap InitalSymbolFlags) + : SymbolFlags(std::move(InitalSymbolFlags)) {} + + virtual ~MaterializationUnit() {} + + /// Return the set of symbols that this source provides. + const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + + /// Called by materialization dispatchers (see + /// ExecutionSession::DispatchMaterializationFunction) to trigger + /// materialization of this MaterializationUnit. + void doMaterialize(VSO &V) { + materialize(MaterializationResponsibility(V, std::move(SymbolFlags))); + } + + /// Called by VSOs to notify MaterializationUnits that the given symbol has + /// been overridden. + void doDiscard(const VSO &V, SymbolStringPtr Name) { + SymbolFlags.erase(Name); + discard(V, std::move(Name)); + } + +protected: + SymbolFlagsMap SymbolFlags; + +private: + virtual void anchor(); + + /// Implementations of this method should materialize all symbols + /// in the materialzation unit, except for those that have been + /// previously discarded. + virtual void materialize(MaterializationResponsibility R) = 0; + + /// Implementations of this method should discard the given symbol + /// from the source (e.g. if the source is an LLVM IR Module and the + /// symbol is a function, delete the function body or mark it available + /// externally). + virtual void discard(const VSO &V, SymbolStringPtr Name) = 0; +}; + +using MaterializationUnitList = + std::vector<std::unique_ptr<MaterializationUnit>>; + +/// A MaterializationUnit implementation for pre-existing absolute symbols. +/// +/// All symbols will be resolved and marked ready as soon as the unit is +/// materialized. +class AbsoluteSymbolsMaterializationUnit : public MaterializationUnit { +public: + AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols); + +private: + void materialize(MaterializationResponsibility R) override; + void discard(const VSO &V, SymbolStringPtr Name) override; + static SymbolFlagsMap extractFlags(const SymbolMap &Symbols); + + SymbolMap Symbols; +}; + +/// Create an AbsoluteSymbolsMaterializationUnit with the given symbols. +/// Useful for inserting absolute symbols into a VSO. E.g.: +/// \code{.cpp} +/// VSO &V = ...; +/// SymbolStringPtr Foo = ...; +/// JITEvaluatedSymbol FooSym = ...; +/// if (auto Err = V.define(absoluteSymbols({{Foo, FooSym}}))) +/// return Err; +/// \endcode +/// +inline std::unique_ptr<AbsoluteSymbolsMaterializationUnit> +absoluteSymbols(SymbolMap Symbols) { + return llvm::make_unique<AbsoluteSymbolsMaterializationUnit>( + std::move(Symbols)); +} + +struct SymbolAliasMapEntry { + SymbolAliasMapEntry() = default; + SymbolAliasMapEntry(SymbolStringPtr Aliasee, JITSymbolFlags AliasFlags) + : Aliasee(std::move(Aliasee)), AliasFlags(AliasFlags) {} + + SymbolStringPtr Aliasee; + JITSymbolFlags AliasFlags; +}; + +/// A map of Symbols to (Symbol, Flags) pairs. +using SymbolAliasMap = std::map<SymbolStringPtr, SymbolAliasMapEntry>; + +/// A materialization unit for symbol aliases. Allows existing symbols to be +/// aliased with alternate flags. +class ReExportsMaterializationUnit : public MaterializationUnit { +public: + /// SourceVSO is allowed to be nullptr, in which case the source VSO is + /// taken to be whatever VSO these definitions are materialized in. This + /// is useful for defining aliases within a VSO. + /// + /// Note: Care must be taken that no sets of aliases form a cycle, as such + /// a cycle will result in a deadlock when any symbol in the cycle is + /// resolved. + ReExportsMaterializationUnit(VSO *SourceVSO, SymbolAliasMap Aliases); + +private: + void materialize(MaterializationResponsibility R) override; + void discard(const VSO &V, SymbolStringPtr Name) override; + static SymbolFlagsMap extractFlags(const SymbolAliasMap &Aliases); + + VSO *SourceVSO = nullptr; + SymbolAliasMap Aliases; +}; + +/// Create a ReExportsMaterializationUnit with the given aliases. +/// Useful for defining symbol aliases.: E.g., given a VSO V containing symbols +/// "foo" and "bar", we can define aliases "baz" (for "foo") and "qux" (for +/// "bar") with: +/// \code{.cpp} +/// SymbolStringPtr Baz = ...; +/// SymbolStringPtr Qux = ...; +/// if (auto Err = V.define(symbolAliases({ +/// {Baz, { Foo, JITSymbolFlags::Exported }}, +/// {Qux, { Bar, JITSymbolFlags::Weak }}})) +/// return Err; +/// \endcode +inline std::unique_ptr<ReExportsMaterializationUnit> +symbolAliases(SymbolAliasMap Aliases) { + return llvm::make_unique<ReExportsMaterializationUnit>(nullptr, + std::move(Aliases)); +} + +/// Create a materialization unit for re-exporting symbols from another VSO +/// with alternative names/flags. +inline std::unique_ptr<ReExportsMaterializationUnit> +reexports(VSO &SourceV, SymbolAliasMap Aliases) { + return llvm::make_unique<ReExportsMaterializationUnit>(&SourceV, + std::move(Aliases)); +} + +/// Build a SymbolAliasMap for the common case where you want to re-export +/// symbols from another VSO with the same linkage/flags. +Expected<SymbolAliasMap> +buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols); + +/// Base utilities for ExecutionSession. +class ExecutionSessionBase { + // FIXME: Remove this when we remove the old ORC layers. + friend class VSO; + +public: + /// For reporting errors. + using ErrorReporter = std::function<void(Error)>; + + /// For dispatching MaterializationUnit::materialize calls. + using DispatchMaterializationFunction = + std::function<void(VSO &V, std::unique_ptr<MaterializationUnit> MU)>; + + /// Construct an ExecutionSessionBase. + /// + /// SymbolStringPools may be shared between ExecutionSessions. + ExecutionSessionBase(std::shared_ptr<SymbolStringPool> SSP = nullptr) + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) {} + + /// Returns the SymbolStringPool for this ExecutionSession. + SymbolStringPool &getSymbolStringPool() const { return *SSP; } + + /// Run the given lambda with the session mutex locked. + template <typename Func> auto runSessionLocked(Func &&F) -> decltype(F()) { + std::lock_guard<std::recursive_mutex> Lock(SessionMutex); + return F(); + } + + /// Set the error reporter function. + ExecutionSessionBase &setErrorReporter(ErrorReporter ReportError) { + this->ReportError = std::move(ReportError); + return *this; + } + + /// Set the materialization dispatch function. + ExecutionSessionBase &setDispatchMaterialization( + DispatchMaterializationFunction DispatchMaterialization) { + this->DispatchMaterialization = std::move(DispatchMaterialization); + return *this; + } + + /// Report a error for this execution session. + /// + /// Unhandled errors can be sent here to log them. + void reportError(Error Err) { ReportError(std::move(Err)); } + + /// Allocate a module key for a new module to add to the JIT. + VModuleKey allocateVModule() { return ++LastKey; } + + /// Return a module key to the ExecutionSession so that it can be + /// re-used. This should only be done once all resources associated + /// with the original key have been released. + void releaseVModule(VModuleKey Key) { /* FIXME: Recycle keys */ + } + + void legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err); + + using LegacyAsyncLookupFunction = std::function<SymbolNameSet( + std::shared_ptr<AsynchronousSymbolQuery> Q, SymbolNameSet Names)>; + + /// A legacy lookup function for JITSymbolResolverAdapter. + /// Do not use -- this will be removed soon. + Expected<SymbolMap> + legacyLookup(ExecutionSessionBase &ES, LegacyAsyncLookupFunction AsyncLookup, + SymbolNameSet Names, bool WaiUntilReady, + RegisterDependenciesFunction RegisterDependencies); + + /// Search the given VSO list for the given symbols. + /// + /// + /// The OnResolve callback will be called once all requested symbols are + /// resolved, or if an error occurs prior to resolution. + /// + /// The OnReady callback will be called once all requested symbols are ready, + /// or if an error occurs after resolution but before all symbols are ready. + /// + /// If all symbols are found, the RegisterDependencies function will be called + /// while the session lock is held. This gives clients a chance to register + /// dependencies for on the queried symbols for any symbols they are + /// materializing (if a MaterializationResponsibility instance is present, + /// this can be implemented by calling + /// MaterializationResponsibility::addDependencies). If there are no + /// dependenant symbols for this query (e.g. it is being made by a top level + /// client to get an address to call) then the value NoDependenciesToRegister + /// can be used. + void lookup(const VSOList &VSOs, const SymbolNameSet &Symbols, + SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + RegisterDependenciesFunction RegisterDependencies); + + /// Blocking version of lookup above. Returns the resolved symbol map. + /// If WaitUntilReady is true (the default), will not return until all + /// requested symbols are ready (or an error occurs). If WaitUntilReady is + /// false, will return as soon as all requested symbols are resolved, + /// or an error occurs. If WaitUntilReady is false and an error occurs + /// after resolution, the function will return a success value, but the + /// error will be reported via reportErrors. + Expected<SymbolMap> lookup(const VSOList &VSOs, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, + bool WaitUntilReady = true); + + /// Materialize the given unit. + void dispatchMaterialization(VSO &V, + std::unique_ptr<MaterializationUnit> MU) { + DispatchMaterialization(V, std::move(MU)); + } + +private: + static void logErrorsToStdErr(Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "JIT session error: "); + } + + static void + materializeOnCurrentThread(VSO &V, std::unique_ptr<MaterializationUnit> MU) { + MU->doMaterialize(V); + } + + void runOutstandingMUs(); + + mutable std::recursive_mutex SessionMutex; + std::shared_ptr<SymbolStringPool> SSP; + VModuleKey LastKey = 0; + ErrorReporter ReportError = logErrorsToStdErr; + DispatchMaterializationFunction DispatchMaterialization = + materializeOnCurrentThread; + + // FIXME: Remove this (and runOutstandingMUs) once the linking layer works + // with callbacks from asynchronous queries. + mutable std::recursive_mutex OutstandingMUsMutex; + std::vector<std::pair<VSO *, std::unique_ptr<MaterializationUnit>>> + OutstandingMUs; +}; + +/// A symbol query that returns results via a callback when results are +/// ready. +/// +/// makes a callback when all symbols are available. +class AsynchronousSymbolQuery { + friend class ExecutionSessionBase; + friend class VSO; + +public: + + /// Create a query for the given symbols, notify-resolved and + /// notify-ready callbacks. + AsynchronousSymbolQuery(const SymbolNameSet &Symbols, + SymbolsResolvedCallback NotifySymbolsResolved, + SymbolsReadyCallback NotifySymbolsReady); + + /// Set the resolved symbol information for the given symbol name. + void resolve(const SymbolStringPtr &Name, JITEvaluatedSymbol Sym); + + /// Returns true if all symbols covered by this query have been + /// resolved. + bool isFullyResolved() const { return NotYetResolvedCount == 0; } + + /// Call the NotifySymbolsResolved callback. + /// + /// This should only be called if all symbols covered by the query have been + /// resolved. + void handleFullyResolved(); + + /// Notify the query that a requested symbol is ready for execution. + void notifySymbolReady(); + + /// Returns true if all symbols covered by this query are ready. + bool isFullyReady() const { return NotYetReadyCount == 0; } + + /// Calls the NotifySymbolsReady callback. + /// + /// This should only be called if all symbols covered by this query are ready. + void handleFullyReady(); + +private: + void addQueryDependence(VSO &V, SymbolStringPtr Name); + + void removeQueryDependence(VSO &V, const SymbolStringPtr &Name); + + bool canStillFail(); + + void handleFailed(Error Err); + + void detach(); + + SymbolsResolvedCallback NotifySymbolsResolved; + SymbolsReadyCallback NotifySymbolsReady; + SymbolDependenceMap QueryRegistrations; + SymbolMap ResolvedSymbols; + size_t NotYetResolvedCount; + size_t NotYetReadyCount; +}; + +/// A symbol table that supports asynchoronous symbol queries. +/// +/// Represents a virtual shared object. Instances can not be copied or moved, so +/// their addresses may be used as keys for resource management. +/// VSO state changes must be made via an ExecutionSession to guarantee that +/// they are synchronized with respect to other VSO operations. +class VSO { + friend class AsynchronousSymbolQuery; + friend class ExecutionSession; + friend class ExecutionSessionBase; + friend class MaterializationResponsibility; +public: + using FallbackDefinitionGeneratorFunction = + std::function<SymbolNameSet(VSO &Parent, const SymbolNameSet &Names)>; + + using AsynchronousSymbolQuerySet = + std::set<std::shared_ptr<AsynchronousSymbolQuery>>; + + VSO(const VSO &) = delete; + VSO &operator=(const VSO &) = delete; + VSO(VSO &&) = delete; + VSO &operator=(VSO &&) = delete; + + /// Get the name for this VSO. + const std::string &getName() const { return VSOName; } + + /// Get a reference to the ExecutionSession for this VSO. + ExecutionSessionBase &getExecutionSession() const { return ES; } + + /// Set a fallback defenition generator. If set, lookup and lookupFlags will + /// pass the unresolved symbols set to the fallback definition generator, + /// allowing it to add a new definition to the VSO. + void setFallbackDefinitionGenerator( + FallbackDefinitionGeneratorFunction FallbackDefinitionGenerator) { + this->FallbackDefinitionGenerator = std::move(FallbackDefinitionGenerator); + } + + /// Set the search order to be used when fixing up definitions in VSO. + /// This will replace the previous search order, and apply to any symbol + /// resolutions made for definitions in this VSO after the call to + /// setSearchOrder (even if the definition itself was added before the + /// call). + /// + /// If SearchThisVSOFirst is set, which by default it is, then this VSO will + /// add itself to the beginning of the SearchOrder (Clients should *not* + /// put this VSO in the list in this case, to avoid redundant lookups). + /// + /// If SearchThisVSOFirst is false then the search order will be used as + /// given. The main motivation for this feature is to support deliberate + /// shadowing of symbols in this VSO by a facade VSO. For example, the + /// facade may resolve function names to stubs, and the stubs may compile + /// lazily by looking up symbols in this dylib. Adding the facade dylib + /// as the first in the search order (instead of this dylib) ensures that + /// definitions within this dylib resolve to the lazy-compiling stubs, + /// rather than immediately materializing the definitions in this dylib. + void setSearchOrder(VSOList NewSearchOrder, bool SearchThisVSOFirst = true); + + /// Add the given VSO to the search order for definitions in this VSO. + void addToSearchOrder(VSO &V); + + /// Replace OldV with NewV in the search order if OldV is present. Otherwise + /// this operation is a no-op. + void replaceInSearchOrder(VSO &OldV, VSO &NewV); + + /// Remove the given VSO from the search order for this VSO if it is + /// present. Otherwise this operation is a no-op. + void removeFromSearchOrder(VSO &V); + + /// Do something with the search order (run under the session lock). + template <typename Func> + auto withSearchOrderDo(Func &&F) + -> decltype(F(std::declval<const VSOList &>())) { + return ES.runSessionLocked([&]() { return F(SearchOrder); }); + } + + /// Define all symbols provided by the materialization unit to be part + /// of the given VSO. + template <typename UniquePtrToMaterializationUnit> + typename std::enable_if< + std::is_convertible< + typename std::decay<UniquePtrToMaterializationUnit>::type, + std::unique_ptr<MaterializationUnit>>::value, + Error>::type + define(UniquePtrToMaterializationUnit &&MU) { + return ES.runSessionLocked([&, this]() -> Error { + assert(MU && "Can't define with a null MU"); + + if (auto Err = defineImpl(*MU)) + return Err; + + /// defineImpl succeeded. + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + for (auto &KV : UMI->MU->getSymbols()) + UnmaterializedInfos[KV.first] = UMI; + + return Error::success(); + }); + } + + /// Search the given VSO for the symbols in Symbols. If found, store + /// the flags for each symbol in Flags. Returns any unresolved symbols. + SymbolFlagsMap lookupFlags(const SymbolNameSet &Names); + + /// Dump current VSO state to OS. + void dump(raw_ostream &OS); + + /// FIXME: Remove this when we remove the old ORC layers. + /// Search the given VSOs in order for the symbols in Symbols. Results + /// (once they become available) will be returned via the given Query. + /// + /// If any symbol is not found then the unresolved symbols will be returned, + /// and the query will not be applied. The Query is not failed and can be + /// re-used in a subsequent lookup once the symbols have been added, or + /// manually failed. + SymbolNameSet legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names); + +private: + using AsynchronousSymbolQueryList = + std::vector<std::shared_ptr<AsynchronousSymbolQuery>>; + + struct UnmaterializedInfo { + UnmaterializedInfo(std::unique_ptr<MaterializationUnit> MU) + : MU(std::move(MU)) {} + + std::unique_ptr<MaterializationUnit> MU; + }; + + using UnmaterializedInfosMap = + std::map<SymbolStringPtr, std::shared_ptr<UnmaterializedInfo>>; + + struct MaterializingInfo { + AsynchronousSymbolQueryList PendingQueries; + SymbolDependenceMap Dependants; + SymbolDependenceMap UnfinalizedDependencies; + bool IsFinalized = false; + }; + + using MaterializingInfosMap = std::map<SymbolStringPtr, MaterializingInfo>; + + using LookupImplActionFlags = enum { + None = 0, + NotifyFullyResolved = 1 << 0U, + NotifyFullyReady = 1 << 1U, + LLVM_MARK_AS_BITMASK_ENUM(NotifyFullyReady) + }; + + VSO(ExecutionSessionBase &ES, std::string Name); + + Error defineImpl(MaterializationUnit &MU); + + SymbolNameSet lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names); + + void lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, MaterializationUnitList &MUs); + + void lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, MaterializationUnitList &MUs); + + LookupImplActionFlags + lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved); + + void detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols); + + void transferFinalizedNodeDependencies(MaterializingInfo &DependantMI, + const SymbolStringPtr &DependantName, + MaterializingInfo &FinalizedMI); + + Error defineMaterializing(const SymbolFlagsMap &SymbolFlags); + + void replace(std::unique_ptr<MaterializationUnit> MU); + + SymbolNameSet getRequestedSymbols(const SymbolFlagsMap &SymbolFlags); + + void addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependants); + + void resolve(const SymbolMap &Resolved); + + void finalize(const SymbolFlagsMap &Finalized); + + void notifyFailed(const SymbolNameSet &FailedSymbols); + + ExecutionSessionBase &ES; + std::string VSOName; + SymbolMap Symbols; + UnmaterializedInfosMap UnmaterializedInfos; + MaterializingInfosMap MaterializingInfos; + FallbackDefinitionGeneratorFunction FallbackDefinitionGenerator; + VSOList SearchOrder; +}; + +/// An ExecutionSession represents a running JIT program. +class ExecutionSession : public ExecutionSessionBase { +public: + using ErrorReporter = std::function<void(Error)>; + + using DispatchMaterializationFunction = + std::function<void(VSO &V, std::unique_ptr<MaterializationUnit> MU)>; + + /// Construct an ExecutionEngine. + /// + /// SymbolStringPools may be shared between ExecutionSessions. + ExecutionSession(std::shared_ptr<SymbolStringPool> SSP = nullptr) + : ExecutionSessionBase(std::move(SSP)) {} + + /// Add a new VSO to this ExecutionSession. + VSO &createVSO(std::string Name); + +private: + std::vector<std::unique_ptr<VSO>> VSOs; +}; + +/// Look up the given names in the given VSOs. +/// VSOs will be searched in order and no VSO pointer may be null. +/// All symbols must be found within the given VSOs or an error +/// will be returned. +Expected<SymbolMap> lookup(const VSOList &VSOs, SymbolNameSet Names); + +/// Look up a symbol by searching a list of VSOs. +Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name); + +/// Mangles symbol names then uniques them in the context of an +/// ExecutionSession. +class MangleAndInterner { +public: + MangleAndInterner(ExecutionSessionBase &ES, const DataLayout &DL); + SymbolStringPtr operator()(StringRef Name); + +private: + ExecutionSessionBase &ES; + const DataLayout &DL; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_CORE_H diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index d9b45c6a1e29..e27f6e1e2cd6 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -17,13 +17,16 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/iterator_range.h" #include "llvm/ExecutionEngine/JITSymbol.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Target/TargetOptions.h" #include <algorithm> #include <cstdint> #include <string> -#include <vector> #include <utility> +#include <vector> namespace llvm { @@ -31,18 +34,58 @@ class ConstantArray; class GlobalVariable; class Function; class Module; +class TargetMachine; class Value; namespace orc { -/// @brief This iterator provides a convenient way to iterate over the elements +/// A utility class for building TargetMachines for JITs. +class JITTargetMachineBuilder { +public: + JITTargetMachineBuilder(Triple TT); + static Expected<JITTargetMachineBuilder> detectHost(); + Expected<std::unique_ptr<TargetMachine>> createTargetMachine(); + + JITTargetMachineBuilder &setArch(std::string Arch) { + this->Arch = std::move(Arch); + return *this; + } + JITTargetMachineBuilder &setCPU(std::string CPU) { + this->CPU = std::move(CPU); + return *this; + } + JITTargetMachineBuilder &setRelocationModel(Optional<Reloc::Model> RM) { + this->RM = std::move(RM); + return *this; + } + JITTargetMachineBuilder &setCodeModel(Optional<CodeModel::Model> CM) { + this->CM = std::move(CM); + return *this; + } + JITTargetMachineBuilder & + addFeatures(const std::vector<std::string> &FeatureVec); + SubtargetFeatures &getFeatures() { return Features; } + TargetOptions &getOptions() { return Options; } + +private: + Triple TT; + std::string Arch; + std::string CPU; + SubtargetFeatures Features; + TargetOptions Options; + Optional<Reloc::Model> RM; + Optional<CodeModel::Model> CM; + CodeGenOpt::Level OptLevel = CodeGenOpt::Default; +}; + +/// This iterator provides a convenient way to iterate over the elements /// of an llvm.global_ctors/llvm.global_dtors instance. /// /// The easiest way to get hold of instances of this class is to use the /// getConstructors/getDestructors functions. class CtorDtorIterator { public: - /// @brief Accessor for an element of the global_ctors/global_dtors array. + /// Accessor for an element of the global_ctors/global_dtors array. /// /// This class provides a read-only view of the element with any casts on /// the function stripped away. @@ -55,23 +98,23 @@ public: Value *Data; }; - /// @brief Construct an iterator instance. If End is true then this iterator + /// Construct an iterator instance. If End is true then this iterator /// acts as the end of the range, otherwise it is the beginning. CtorDtorIterator(const GlobalVariable *GV, bool End); - /// @brief Test iterators for equality. + /// Test iterators for equality. bool operator==(const CtorDtorIterator &Other) const; - /// @brief Test iterators for inequality. + /// Test iterators for inequality. bool operator!=(const CtorDtorIterator &Other) const; - /// @brief Pre-increment iterator. + /// Pre-increment iterator. CtorDtorIterator& operator++(); - /// @brief Post-increment iterator. + /// Post-increment iterator. CtorDtorIterator operator++(int); - /// @brief Dereference iterator. The resulting value provides a read-only view + /// Dereference iterator. The resulting value provides a read-only view /// of this element of the global_ctors/global_dtors list. Element operator*() const; @@ -80,32 +123,31 @@ private: unsigned I; }; -/// @brief Create an iterator range over the entries of the llvm.global_ctors +/// Create an iterator range over the entries of the llvm.global_ctors /// array. iterator_range<CtorDtorIterator> getConstructors(const Module &M); -/// @brief Create an iterator range over the entries of the llvm.global_ctors +/// Create an iterator range over the entries of the llvm.global_ctors /// array. iterator_range<CtorDtorIterator> getDestructors(const Module &M); -/// @brief Convenience class for recording constructor/destructor names for +/// Convenience class for recording constructor/destructor names for /// later execution. template <typename JITLayerT> class CtorDtorRunner { public: - /// @brief Construct a CtorDtorRunner for the given range using the given + /// Construct a CtorDtorRunner for the given range using the given /// name mangling function. - CtorDtorRunner(std::vector<std::string> CtorDtorNames, - typename JITLayerT::ModuleHandleT H) - : CtorDtorNames(std::move(CtorDtorNames)), H(H) {} + CtorDtorRunner(std::vector<std::string> CtorDtorNames, VModuleKey K) + : CtorDtorNames(std::move(CtorDtorNames)), K(K) {} - /// @brief Run the recorded constructors/destructors through the given JIT + /// Run the recorded constructors/destructors through the given JIT /// layer. Error runViaLayer(JITLayerT &JITLayer) const { using CtorDtorTy = void (*)(); - for (const auto &CtorDtorName : CtorDtorNames) - if (auto CtorDtorSym = JITLayer.findSymbolIn(H, CtorDtorName, false)) { + for (const auto &CtorDtorName : CtorDtorNames) { + if (auto CtorDtorSym = JITLayer.findSymbolIn(K, CtorDtorName, false)) { if (auto AddrOrErr = CtorDtorSym.getAddress()) { CtorDtorTy CtorDtor = reinterpret_cast<CtorDtorTy>(static_cast<uintptr_t>(*AddrOrErr)); @@ -118,15 +160,30 @@ public: else return make_error<JITSymbolNotFound>(CtorDtorName); } + } return Error::success(); } private: std::vector<std::string> CtorDtorNames; - typename JITLayerT::ModuleHandleT H; + orc::VModuleKey K; }; -/// @brief Support class for static dtor execution. For hosted (in-process) JITs +class CtorDtorRunner2 { +public: + CtorDtorRunner2(VSO &V) : V(V) {} + void add(iterator_range<CtorDtorIterator> CtorDtors); + Error run(); + +private: + using CtorDtorList = std::vector<SymbolStringPtr>; + using CtorDtorPriorityMap = std::map<unsigned, CtorDtorList>; + + VSO &V; + CtorDtorPriorityMap CtorDtorsByPriority; +}; + +/// Support class for static dtor execution. For hosted (in-process) JITs /// only! /// /// If a __cxa_atexit function isn't found C++ programs that use static @@ -141,7 +198,26 @@ private: /// the client determines that destructors should be run (generally at JIT /// teardown or after a return from main), the runDestructors method should be /// called. -class LocalCXXRuntimeOverrides { +class LocalCXXRuntimeOverridesBase { +public: + /// Run any destructors recorded by the overriden __cxa_atexit function + /// (CXAAtExitOverride). + void runDestructors(); + +protected: + template <typename PtrTy> JITTargetAddress toTargetAddress(PtrTy *P) { + return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(P)); + } + + using DestructorPtr = void (*)(void *); + using CXXDestructorDataPair = std::pair<DestructorPtr, void *>; + using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>; + CXXDestructorDataPairList DSOHandleOverride; + static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg, + void *DSOHandle); +}; + +class LocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase { public: /// Create a runtime-overrides class. template <typename MangleFtorT> @@ -158,32 +234,38 @@ public: return nullptr; } - /// Run any destructors recorded by the overriden __cxa_atexit function - /// (CXAAtExitOverride). - void runDestructors(); - private: - template <typename PtrTy> - JITTargetAddress toTargetAddress(PtrTy* P) { - return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(P)); - } - void addOverride(const std::string &Name, JITTargetAddress Addr) { CXXRuntimeOverrides.insert(std::make_pair(Name, Addr)); } StringMap<JITTargetAddress> CXXRuntimeOverrides; +}; - using DestructorPtr = void (*)(void *); - using CXXDestructorDataPair = std::pair<DestructorPtr, void *>; - using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>; - CXXDestructorDataPairList DSOHandleOverride; - static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg, - void *DSOHandle); +class LocalCXXRuntimeOverrides2 : public LocalCXXRuntimeOverridesBase { +public: + Error enable(VSO &V, MangleAndInterner &Mangler); }; -} // end namespace orc +/// A utility class to expose symbols found via dlsym to the JIT. +/// +/// If an instance of this class is attached to a VSO as a fallback definition +/// generator, then any symbol found in the given DynamicLibrary that passes +/// the 'Allow' predicate will be added to the VSO. +class DynamicLibraryFallbackGenerator { +public: + using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + DynamicLibraryFallbackGenerator(sys::DynamicLibrary Dylib, + const DataLayout &DL, SymbolPredicate Allow); + SymbolNameSet operator()(VSO &V, const SymbolNameSet &Names); +private: + sys::DynamicLibrary Dylib; + SymbolPredicate Allow; + char GlobalPrefix; +}; + +} // end namespace orc } // end namespace llvm #endif // LLVM_EXECUTIONENGINE_ORC_EXECUTIONUTILS_H diff --git a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h index 8a48c36f4141..a8a88d7cb2d2 100644 --- a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h @@ -27,7 +27,7 @@ class JITSymbolResolver; namespace orc { -/// @brief Global mapping layer. +/// Global mapping layer. /// /// This layer overrides the findSymbol method to first search a local symbol /// table that the client can define. It can be used to inject new symbol @@ -38,13 +38,13 @@ template <typename BaseLayerT> class GlobalMappingLayer { public: - /// @brief Handle to an added module. + /// Handle to an added module. using ModuleHandleT = typename BaseLayerT::ModuleHandleT; - /// @brief Construct an GlobalMappingLayer with the given BaseLayer + /// Construct an GlobalMappingLayer with the given BaseLayer GlobalMappingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} - /// @brief Add the given module to the JIT. + /// Add the given module to the JIT. /// @return A handle for the added modules. Expected<ModuleHandleT> addModule(std::shared_ptr<Module> M, @@ -52,20 +52,20 @@ public: return BaseLayer.addModule(std::move(M), std::move(Resolver)); } - /// @brief Remove the module set associated with the handle H. + /// Remove the module set associated with the handle H. Error removeModule(ModuleHandleT H) { return BaseLayer.removeModule(H); } - /// @brief Manually set the address to return for the given symbol. + /// Manually set the address to return for the given symbol. void setGlobalMapping(const std::string &Name, JITTargetAddress Addr) { SymbolTable[Name] = Addr; } - /// @brief Remove the given symbol from the global mapping. + /// Remove the given symbol from the global mapping. void eraseGlobalMapping(const std::string &Name) { SymbolTable.erase(Name); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// /// This method will first search the local symbol table, returning /// any symbol found there. If the symbol is not found in the local @@ -81,7 +81,7 @@ public: return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of the given symbol in the context of the of the + /// Get the address of the given symbol in the context of the of the /// module represented by the handle H. This call is forwarded to the /// base layer's implementation. /// @param H The handle for the module to search in. @@ -94,7 +94,7 @@ public: return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); } - /// @brief Immediately emit and finalize the module set represented by the + /// Immediately emit and finalize the module set represented by the /// given handle. /// @param H Handle for module set to emit/finalize. Error emitAndFinalize(ModuleHandleT H) { diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index fadd334bed0f..ad6481548d59 100644 --- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -16,7 +16,9 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include <memory> #include <string> @@ -26,7 +28,30 @@ class Module; namespace orc { -/// @brief Eager IR compiling layer. +class IRCompileLayer2 : public IRLayer { +public: + using CompileFunction = + std::function<Expected<std::unique_ptr<MemoryBuffer>>(Module &)>; + + using NotifyCompiledFunction = + std::function<void(VModuleKey K, std::unique_ptr<Module>)>; + + IRCompileLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer, + CompileFunction Compile); + + void setNotifyCompiled(NotifyCompiledFunction NotifyCompiled); + + void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) override; + +private: + mutable std::mutex IRLayerMutex; + ObjectLayer &BaseLayer; + CompileFunction Compile; + NotifyCompiledFunction NotifyCompiled = NotifyCompiledFunction(); +}; + +/// Eager IR compiling layer. /// /// This layer immediately compiles each IR module added via addModule to an /// object file and adds this module file to the layer below, which must @@ -34,36 +59,40 @@ namespace orc { template <typename BaseLayerT, typename CompileFtor> class IRCompileLayer { public: + /// Callback type for notifications when modules are compiled. + using NotifyCompiledCallback = + std::function<void(VModuleKey K, std::unique_ptr<Module>)>; - /// @brief Handle to a compiled module. - using ModuleHandleT = typename BaseLayerT::ObjHandleT; - - /// @brief Construct an IRCompileLayer with the given BaseLayer, which must + /// Construct an IRCompileLayer with the given BaseLayer, which must /// implement the ObjectLayer concept. - IRCompileLayer(BaseLayerT &BaseLayer, CompileFtor Compile) - : BaseLayer(BaseLayer), Compile(std::move(Compile)) {} + IRCompileLayer( + BaseLayerT &BaseLayer, CompileFtor Compile, + NotifyCompiledCallback NotifyCompiled = NotifyCompiledCallback()) + : BaseLayer(BaseLayer), Compile(std::move(Compile)), + NotifyCompiled(std::move(NotifyCompiled)) {} - /// @brief Get a reference to the compiler functor. + /// Get a reference to the compiler functor. CompileFtor& getCompiler() { return Compile; } - /// @brief Compile the module, and add the resulting object to the base layer - /// along with the given memory manager and symbol resolver. - /// - /// @return A handle for the added module. - Expected<ModuleHandleT> - addModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) { - using CompileResult = decltype(Compile(*M)); - auto Obj = std::make_shared<CompileResult>(Compile(*M)); - return BaseLayer.addObject(std::move(Obj), std::move(Resolver)); + /// (Re)set the NotifyCompiled callback. + void setNotifyCompiled(NotifyCompiledCallback NotifyCompiled) { + this->NotifyCompiled = std::move(NotifyCompiled); } - /// @brief Remove the module associated with the handle H. - Error removeModule(ModuleHandleT H) { - return BaseLayer.removeObject(H); + /// Compile the module, and add the resulting object to the base layer + /// along with the given memory manager and symbol resolver. + Error addModule(VModuleKey K, std::unique_ptr<Module> M) { + if (auto Err = BaseLayer.addObject(std::move(K), Compile(*M))) + return Err; + if (NotifyCompiled) + NotifyCompiled(std::move(K), std::move(M)); + return Error::success(); } - /// @brief Search for the given named symbol. + /// Remove the module associated with the VModuleKey K. + Error removeModule(VModuleKey K) { return BaseLayer.removeObject(K); } + + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. @@ -71,29 +100,28 @@ public: return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of the given symbol in compiled module represented + /// Get the address of the given symbol in compiled module represented /// by the handle H. This call is forwarded to the base layer's /// implementation. - /// @param H The handle for the module to search in. + /// @param K The VModuleKey for the module to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the /// given module. - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + JITSymbol findSymbolIn(VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); + return BaseLayer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - /// @brief Immediately emit and finalize the module represented by the given + /// Immediately emit and finalize the module represented by the given /// handle. - /// @param H Handle for module to emit/finalize. - Error emitAndFinalize(ModuleHandleT H) { - return BaseLayer.emitAndFinalize(H); - } + /// @param K The VModuleKey for the module to emit/finalize. + Error emitAndFinalize(VModuleKey K) { return BaseLayer.emitAndFinalize(K); } private: BaseLayerT &BaseLayer; CompileFtor Compile; + NotifyCompiledCallback NotifyCompiled; }; } // end namespace orc diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index 476061afda59..266a0f45b3e4 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -15,6 +15,7 @@ #define LLVM_EXECUTIONENGINE_ORC_IRTRANSFORMLAYER_H #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" #include <memory> #include <string> @@ -22,7 +23,32 @@ namespace llvm { class Module; namespace orc { -/// @brief IR mutating layer. +class IRTransformLayer2 : public IRLayer { +public: + + using TransformFunction = + std::function<Expected<std::unique_ptr<Module>>(std::unique_ptr<Module>)>; + + IRTransformLayer2(ExecutionSession &ES, IRLayer &BaseLayer, + TransformFunction Transform = identityTransform); + + void setTransform(TransformFunction Transform) { + this->Transform = std::move(Transform); + } + + void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) override; + + static std::unique_ptr<Module> identityTransform(std::unique_ptr<Module> M) { + return M; + } + +private: + IRLayer &BaseLayer; + TransformFunction Transform; +}; + +/// IR mutating layer. /// /// This layer applies a user supplied transform to each module that is added, /// then adds the transformed module to the layer below. @@ -30,28 +56,23 @@ template <typename BaseLayerT, typename TransformFtor> class IRTransformLayer { public: - /// @brief Handle to a set of added modules. - using ModuleHandleT = typename BaseLayerT::ModuleHandleT; - - /// @brief Construct an IRTransformLayer with the given BaseLayer + /// Construct an IRTransformLayer with the given BaseLayer IRTransformLayer(BaseLayerT &BaseLayer, TransformFtor Transform = TransformFtor()) : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} - /// @brief Apply the transform functor to the module, then add the module to + /// Apply the transform functor to the module, then add the module to /// the layer below, along with the memory manager and symbol resolver. /// /// @return A handle for the added modules. - Expected<ModuleHandleT> - addModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) { - return BaseLayer.addModule(Transform(std::move(M)), std::move(Resolver)); + Error addModule(VModuleKey K, std::unique_ptr<Module> M) { + return BaseLayer.addModule(std::move(K), Transform(std::move(M))); } - /// @brief Remove the module associated with the handle H. - Error removeModule(ModuleHandleT H) { return BaseLayer.removeModule(H); } + /// Remove the module associated with the VModuleKey K. + Error removeModule(VModuleKey K) { return BaseLayer.removeModule(K); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. @@ -59,30 +80,28 @@ public: return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of the given symbol in the context of the module - /// represented by the handle H. This call is forwarded to the base + /// Get the address of the given symbol in the context of the module + /// represented by the VModuleKey K. This call is forwarded to the base /// layer's implementation. - /// @param H The handle for the module to search in. + /// @param K The VModuleKey for the module to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the /// given module. - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + JITSymbol findSymbolIn(VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); + return BaseLayer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - /// @brief Immediately emit and finalize the module represented by the given - /// handle. - /// @param H Handle for module to emit/finalize. - Error emitAndFinalize(ModuleHandleT H) { - return BaseLayer.emitAndFinalize(H); - } + /// Immediately emit and finalize the module represented by the given + /// VModuleKey. + /// @param K The VModuleKey for the module to emit/finalize. + Error emitAndFinalize(VModuleKey K) { return BaseLayer.emitAndFinalize(K); } - /// @brief Access the transform functor directly. + /// Access the transform functor directly. TransformFtor& getTransform() { return Transform; } - /// @brief Access the mumate functor directly. + /// Access the mumate functor directly. const TransformFtor& getTransform() const { return Transform; } private: diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index 029b86a6d2ca..8b0b3fdb7df4 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/Support/Error.h" #include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" @@ -46,98 +47,29 @@ class Value; namespace orc { -/// @brief Target-independent base class for compile callback management. +/// Target-independent base class for compile callback management. class JITCompileCallbackManager { public: - using CompileFtor = std::function<JITTargetAddress()>; - - /// @brief Handle to a newly created compile callback. Can be used to get an - /// IR constant representing the address of the trampoline, and to set - /// the compile action for the callback. - class CompileCallbackInfo { - public: - CompileCallbackInfo(JITTargetAddress Addr, CompileFtor &Compile) - : Addr(Addr), Compile(Compile) {} - - JITTargetAddress getAddress() const { return Addr; } - void setCompileAction(CompileFtor Compile) { - this->Compile = std::move(Compile); - } - - private: - JITTargetAddress Addr; - CompileFtor &Compile; - }; - - /// @brief Construct a JITCompileCallbackManager. + using CompileFunction = std::function<JITTargetAddress()>; + + /// Construct a JITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - JITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress) - : ErrorHandlerAddress(ErrorHandlerAddress) {} + JITCompileCallbackManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) + : ES(ES), CallbacksVSO(ES.createVSO("<Callbacks>")), + ErrorHandlerAddress(ErrorHandlerAddress) {} virtual ~JITCompileCallbackManager() = default; - /// @brief Execute the callback for the given trampoline id. Called by the JIT - /// to compile functions on demand. - JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - // FIXME: Also raise an error in the Orc error-handler when we finally have - // one. - if (I == ActiveTrampolines.end()) - return ErrorHandlerAddress; - - // Found a callback handler. Yank this trampoline out of the active list and - // put it back in the available trampolines list, then try to run the - // handler's compile and update actions. - // Moving the trampoline ID back to the available list first means there's - // at - // least one available trampoline if the compile action triggers a request - // for - // a new one. - auto Compile = std::move(I->second); - ActiveTrampolines.erase(I); - AvailableTrampolines.push_back(TrampolineAddr); - - if (auto Addr = Compile()) - return Addr; - - return ErrorHandlerAddress; - } - - /// @brief Reserve a compile callback. - Expected<CompileCallbackInfo> getCompileCallback() { - if (auto TrampolineAddrOrErr = getAvailableTrampolineAddr()) { - const auto &TrampolineAddr = *TrampolineAddrOrErr; - auto &Compile = this->ActiveTrampolines[TrampolineAddr]; - return CompileCallbackInfo(TrampolineAddr, Compile); - } else - return TrampolineAddrOrErr.takeError(); - } - - /// @brief Get a CompileCallbackInfo for an existing callback. - CompileCallbackInfo getCompileCallbackInfo(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - assert(I != ActiveTrampolines.end() && "Not an active trampoline."); - return CompileCallbackInfo(I->first, I->second); - } + /// Reserve a compile callback. + Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile); - /// @brief Release a compile callback. - /// - /// Note: Callbacks are auto-released after they execute. This method should - /// only be called to manually release a callback that is not going to - /// execute. - void releaseCompileCallback(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - assert(I != ActiveTrampolines.end() && "Not an active trampoline."); - ActiveTrampolines.erase(I); - AvailableTrampolines.push_back(TrampolineAddr); - } + /// Execute the callback for the given trampoline id. Called by the JIT + /// to compile functions on demand. + JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr); protected: - JITTargetAddress ErrorHandlerAddress; - - using TrampolineMapT = std::map<JITTargetAddress, CompileFtor>; - TrampolineMapT ActiveTrampolines; std::vector<JITTargetAddress> AvailableTrampolines; private: @@ -156,17 +88,25 @@ private: virtual Error grow() = 0; virtual void anchor(); + + std::mutex CCMgrMutex; + ExecutionSession &ES; + VSO &CallbacksVSO; + JITTargetAddress ErrorHandlerAddress; + std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol; + size_t NextCallbackId = 0; }; -/// @brief Manage compile callbacks for in-process JITs. +/// Manage compile callbacks for in-process JITs. template <typename TargetT> class LocalJITCompileCallbackManager : public JITCompileCallbackManager { public: - /// @brief Construct a InProcessJITCompileCallbackManager. + /// Construct a InProcessJITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - LocalJITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress) - : JITCompileCallbackManager(ErrorHandlerAddress) { + LocalJITCompileCallbackManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) + : JITCompileCallbackManager(ES, ErrorHandlerAddress) { /// Set up the resolver block. std::error_code EC; ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( @@ -229,38 +169,38 @@ private: std::vector<sys::OwningMemoryBlock> TrampolineBlocks; }; -/// @brief Base class for managing collections of named indirect stubs. +/// Base class for managing collections of named indirect stubs. class IndirectStubsManager { public: - /// @brief Map type for initializing the manager. See init. + /// Map type for initializing the manager. See init. using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>; virtual ~IndirectStubsManager() = default; - /// @brief Create a single stub with the given name, target address and flags. + /// Create a single stub with the given name, target address and flags. virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags) = 0; - /// @brief Create StubInits.size() stubs with the given names, target + /// Create StubInits.size() stubs with the given names, target /// addresses, and flags. virtual Error createStubs(const StubInitsMap &StubInits) = 0; - /// @brief Find the stub with the given name. If ExportedStubsOnly is true, + /// Find the stub with the given name. If ExportedStubsOnly is true, /// this will only return a result if the stub's flags indicate that it /// is exported. - virtual JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; + virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; - /// @brief Find the implementation-pointer for the stub. - virtual JITSymbol findPointer(StringRef Name) = 0; + /// Find the implementation-pointer for the stub. + virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0; - /// @brief Change the value of the implementation pointer for the stub. + /// Change the value of the implementation pointer for the stub. virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0; private: virtual void anchor(); }; -/// @brief IndirectStubsManager implementation for the host architecture, e.g. +/// IndirectStubsManager implementation for the host architecture, e.g. /// OrcX86_64. (See OrcArchitectureSupport.h). template <typename TargetT> class LocalIndirectStubsManager : public IndirectStubsManager { @@ -286,7 +226,7 @@ public: return Error::success(); } - JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { auto I = StubIndexes.find(Name); if (I == StubIndexes.end()) return nullptr; @@ -295,13 +235,13 @@ public: assert(StubAddr && "Missing stub address"); auto StubTargetAddr = static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr)); - auto StubSymbol = JITSymbol(StubTargetAddr, I->second.second); + auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second); if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) return nullptr; return StubSymbol; } - JITSymbol findPointer(StringRef Name) override { + JITEvaluatedSymbol findPointer(StringRef Name) override { auto I = StubIndexes.find(Name); if (I == StubIndexes.end()) return nullptr; @@ -310,7 +250,7 @@ public: assert(PtrAddr && "Missing pointer address"); auto PtrTargetAddr = static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr)); - return JITSymbol(PtrTargetAddr, I->second.second); + return JITEvaluatedSymbol(PtrTargetAddr, I->second.second); } Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { @@ -354,45 +294,45 @@ private: StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; }; -/// @brief Create a local compile callback manager. +/// Create a local compile callback manager. /// /// The given target triple will determine the ABI, and the given /// ErrorHandlerAddress will be used by the resulting compile callback /// manager if a compile callback fails. std::unique_ptr<JITCompileCallbackManager> -createLocalCompileCallbackManager(const Triple &T, +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress); -/// @brief Create a local indriect stubs manager builder. +/// Create a local indriect stubs manager builder. /// /// The given target triple will determine the ABI. std::function<std::unique_ptr<IndirectStubsManager>()> createLocalIndirectStubsManagerBuilder(const Triple &T); -/// @brief Build a function pointer of FunctionType with the given constant +/// Build a function pointer of FunctionType with the given constant /// address. /// /// Usage example: Turn a trampoline address into a function pointer constant /// for use in a stub. Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr); -/// @brief Create a function pointer with the given type, name, and initializer +/// Create a function pointer with the given type, name, and initializer /// in the given Module. GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name, Constant *Initializer); -/// @brief Turn a function declaration into a stub function that makes an +/// Turn a function declaration into a stub function that makes an /// indirect call using the given function pointer. void makeStub(Function &F, Value &ImplPointer); -/// @brief Raise linkage types and rename as necessary to ensure that all +/// Raise linkage types and rename as necessary to ensure that all /// symbols are accessible for other modules. /// /// This should be called before partitioning a module to ensure that the /// partitions retain access to each other's symbols. void makeAllSymbolsExternallyAccessible(Module &M); -/// @brief Clone a function declaration into a new module. +/// Clone a function declaration into a new module. /// /// This function can be used as the first step towards creating a callback /// stub (see makeStub), or moving a function body (see moveFunctionBody). @@ -407,7 +347,7 @@ void makeAllSymbolsExternallyAccessible(Module &M); Function *cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap = nullptr); -/// @brief Move the body of function 'F' to a cloned function declaration in a +/// Move the body of function 'F' to a cloned function declaration in a /// different module (See related cloneFunctionDecl). /// /// If the target function declaration is not supplied via the NewF parameter @@ -419,11 +359,11 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, ValueMaterializer *Materializer = nullptr, Function *NewF = nullptr); -/// @brief Clone a global variable declaration into a new module. +/// Clone a global variable declaration into a new module. GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap = nullptr); -/// @brief Move global variable GV from its parent module to cloned global +/// Move global variable GV from its parent module to cloned global /// declaration in a different module. /// /// If the target global declaration is not supplied via the NewGV parameter @@ -436,11 +376,11 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV, ValueMaterializer *Materializer = nullptr, GlobalVariable *NewGV = nullptr); -/// @brief Clone a global alias declaration into a new module. +/// Clone a global alias declaration into a new module. GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap); -/// @brief Clone module flags metadata into the destination module. +/// Clone module flags metadata into the destination module. void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, ValueToValueMapTy &VMap); diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h new file mode 100644 index 000000000000..df655bd82006 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -0,0 +1,143 @@ +//===----- LLJIT.h -- An ORC-based JIT for compiling LLVM IR ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for 3Bdetails. +// +//===----------------------------------------------------------------------===// +// +// An ORC-based JIT for compiling LLVM IR. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LLJIT_H +#define LLVM_EXECUTIONENGINE_ORC_LLJIT_H + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/Target/TargetMachine.h" + +namespace llvm { +namespace orc { + +/// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT. +class LLJIT { +public: + /// Create an LLJIT instance. + static Expected<std::unique_ptr<LLJIT>> + Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL); + + /// Returns a reference to the ExecutionSession for this JIT instance. + ExecutionSession &getExecutionSession() { return *ES; } + + /// Returns a reference to the VSO representing the JIT'd main program. + VSO &getMainVSO() { return Main; } + + /// Convenience method for defining an absolute symbol. + Error defineAbsolute(StringRef Name, JITEvaluatedSymbol Address); + + /// Adds an IR module to the given VSO. + Error addIRModule(VSO &V, std::unique_ptr<Module> M); + + /// Adds an IR module to the Main VSO. + Error addIRModule(std::unique_ptr<Module> M) { + return addIRModule(Main, std::move(M)); + } + + /// Look up a symbol in VSO V by the symbol's linker-mangled name (to look up + /// symbols based on their IR name use the lookup function instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(VSO &V, StringRef Name); + + /// Look up a symbol in the main VSO by the symbol's linker-mangled name (to + /// look up symbols based on their IR name use the lookup function instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(StringRef Name) { + return lookupLinkerMangled(Main, Name); + } + + /// Look up a symbol in VSO V based on its IR symbol name. + Expected<JITEvaluatedSymbol> lookup(VSO &V, StringRef UnmangledName) { + return lookupLinkerMangled(V, mangle(UnmangledName)); + } + + /// Look up a symbol in the main VSO based on its IR symbol name. + Expected<JITEvaluatedSymbol> lookup(StringRef UnmangledName) { + return lookup(Main, UnmangledName); + } + + /// Runs all not-yet-run static constructors. + Error runConstructors() { return CtorRunner.run(); } + + /// Runs all not-yet-run static destructors. + Error runDestructors() { return DtorRunner.run(); } + +protected: + LLJIT(std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, + DataLayout DL); + + std::shared_ptr<RuntimeDyld::MemoryManager> getMemoryManager(VModuleKey K); + + std::string mangle(StringRef UnmangledName); + + Error applyDataLayout(Module &M); + + void recordCtorDtors(Module &M); + + std::unique_ptr<ExecutionSession> ES; + VSO &Main; + + std::unique_ptr<TargetMachine> TM; + DataLayout DL; + + RTDyldObjectLinkingLayer2 ObjLinkingLayer; + IRCompileLayer2 CompileLayer; + + CtorDtorRunner2 CtorRunner, DtorRunner; +}; + +/// An extended version of LLJIT that supports lazy function-at-a-time +/// compilation of LLVM IR. +class LLLazyJIT : public LLJIT { +public: + /// Create an LLLazyJIT instance. + static Expected<std::unique_ptr<LLLazyJIT>> + Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL, LLVMContext &Ctx); + + /// Set an IR transform (e.g. pass manager pipeline) to run on each function + /// when it is compiled. + void setLazyCompileTransform(IRTransformLayer2::TransformFunction Transform) { + TransformLayer.setTransform(std::move(Transform)); + } + + /// Add a module to be lazily compiled to VSO V. + Error addLazyIRModule(VSO &V, std::unique_ptr<Module> M); + + /// Add a module to be lazily compiled to the main VSO. + Error addLazyIRModule(std::unique_ptr<Module> M) { + return addLazyIRModule(Main, std::move(M)); + } + +private: + LLLazyJIT(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL, LLVMContext &Ctx, + std::unique_ptr<JITCompileCallbackManager> CCMgr, + std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder); + + std::unique_ptr<JITCompileCallbackManager> CCMgr; + std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder; + + IRTransformLayer2 TransformLayer; + CompileOnDemandLayer2 CODLayer; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LLJIT_H diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 228392ae0d4a..7b6f3d2f92ab 100644 --- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -23,7 +23,7 @@ namespace llvm { namespace orc { template <typename DylibLookupFtorT, typename ExternalLookupFtorT> -class LambdaResolver : public JITSymbolResolver { +class LambdaResolver : public LegacyJITSymbolResolver { public: LambdaResolver(DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) diff --git a/include/llvm/ExecutionEngine/Orc/Layer.h b/include/llvm/ExecutionEngine/Orc/Layer.h new file mode 100644 index 000000000000..91bd4fb83e6f --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Layer.h @@ -0,0 +1,129 @@ +//===---------------- Layer.h -- Layer interfaces --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Layer interfaces. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LAYER_H +#define LLVM_EXECUTIONENGINE_ORC_LAYER_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/IR/Module.h" + +namespace llvm { +namespace orc { + +/// Interface for layers that accept LLVM IR. +class IRLayer { +public: + IRLayer(ExecutionSession &ES); + virtual ~IRLayer(); + + /// Returns the ExecutionSession for this layer. + ExecutionSession &getExecutionSession() { return ES; } + + /// Adds a MaterializationUnit representing the given IR to the given VSO. + virtual Error add(VSO &V, VModuleKey K, std::unique_ptr<Module> M); + + /// Emit should materialize the given IR. + virtual void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) = 0; + +private: + ExecutionSession &ES; +}; + +/// IRMaterializationUnit is a convenient base class for MaterializationUnits +/// wrapping LLVM IR. Represents materialization responsibility for all symbols +/// in the given module. If symbols are overridden by other definitions, then +/// their linkage is changed to available-externally. +class IRMaterializationUnit : public MaterializationUnit { +public: + using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; + + /// Create an IRMaterializationLayer. Scans the module to build the + /// SymbolFlags and SymbolToDefinition maps. + IRMaterializationUnit(ExecutionSession &ES, std::unique_ptr<Module> M); + + /// Create an IRMaterializationLayer from a module, and pre-existing + /// SymbolFlags and SymbolToDefinition maps. The maps must provide + /// entries for each definition in M. + /// This constructor is useful for delegating work from one + /// IRMaterializationUnit to another. + IRMaterializationUnit(std::unique_ptr<Module> M, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition); + +protected: + std::unique_ptr<Module> M; + SymbolNameToDefinitionMap SymbolToDefinition; + +private: + void discard(const VSO &V, SymbolStringPtr Name) override; +}; + +/// MaterializationUnit that materializes modules by calling the 'emit' method +/// on the given IRLayer. +class BasicIRLayerMaterializationUnit : public IRMaterializationUnit { +public: + BasicIRLayerMaterializationUnit(IRLayer &L, VModuleKey K, + std::unique_ptr<Module> M); +private: + + void materialize(MaterializationResponsibility R) override; + + IRLayer &L; + VModuleKey K; +}; + +/// Interface for Layers that accept object files. +class ObjectLayer { +public: + ObjectLayer(ExecutionSession &ES); + virtual ~ObjectLayer(); + + /// Returns the execution session for this layer. + ExecutionSession &getExecutionSession() { return ES; } + + /// Adds a MaterializationUnit representing the given IR to the given VSO. + virtual Error add(VSO &V, VModuleKey K, std::unique_ptr<MemoryBuffer> O); + + /// Emit should materialize the given IR. + virtual void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) = 0; + +private: + ExecutionSession &ES; +}; + +/// Materializes the given object file (represented by a MemoryBuffer +/// instance) by calling 'emit' on the given ObjectLayer. +class BasicObjectLayerMaterializationUnit : public MaterializationUnit { +public: + + + /// The MemoryBuffer should represent a valid object file. + /// If there is any chance that the file is invalid it should be validated + /// prior to constructing a BasicObjectLayerMaterializationUnit. + BasicObjectLayerMaterializationUnit(ObjectLayer &L, VModuleKey K, + std::unique_ptr<MemoryBuffer> O); + +private: + void materialize(MaterializationResponsibility R) override; + void discard(const VSO &V, SymbolStringPtr Name) override; + + ObjectLayer &L; + VModuleKey K; + std::unique_ptr<MemoryBuffer> O; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LAYER_H diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index b7e462e85d9d..46761b0ca7e1 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" @@ -32,23 +33,18 @@ namespace llvm { namespace orc { -/// @brief Lazy-emitting IR layer. +/// Lazy-emitting IR layer. /// /// This layer accepts LLVM IR Modules (via addModule), but does not /// immediately emit them the layer below. Instead, emissing to the base layer /// is deferred until the first time the client requests the address (via /// JITSymbol::getAddress) for a symbol contained in this layer. template <typename BaseLayerT> class LazyEmittingLayer { -public: - - using BaseLayerHandleT = typename BaseLayerT::ModuleHandleT; - private: class EmissionDeferredModule { public: - EmissionDeferredModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) - : M(std::move(M)), Resolver(std::move(Resolver)) {} + EmissionDeferredModule(VModuleKey K, std::unique_ptr<Module> M) + : K(std::move(K)), M(std::move(M)) {} JITSymbol find(StringRef Name, bool ExportedSymbolsOnly, BaseLayerT &B) { switch (EmitState) { @@ -65,13 +61,11 @@ private: return 0; else if (this->EmitState == NotEmitted) { this->EmitState = Emitting; - if (auto HandleOrErr = this->emitToBaseLayer(B)) - Handle = std::move(*HandleOrErr); - else - return HandleOrErr.takeError(); + if (auto Err = this->emitToBaseLayer(B)) + return std::move(Err); this->EmitState = Emitted; } - if (auto Sym = B.findSymbolIn(Handle, PName, ExportedSymbolsOnly)) + if (auto Sym = B.findSymbolIn(K, PName, ExportedSymbolsOnly)) return Sym.getAddress(); else if (auto Err = Sym.takeError()) return std::move(Err); @@ -89,13 +83,13 @@ private: // RuntimeDyld that did the lookup), so just return a nullptr here. return nullptr; case Emitted: - return B.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + return B.findSymbolIn(K, Name, ExportedSymbolsOnly); } llvm_unreachable("Invalid emit-state."); } Error removeModuleFromBaseLayer(BaseLayerT& BaseLayer) { - return EmitState != NotEmitted ? BaseLayer.removeModule(Handle) + return EmitState != NotEmitted ? BaseLayer.removeModule(K) : Error::success(); } @@ -104,10 +98,10 @@ private: "Cannot emitAndFinalize while already emitting"); if (EmitState == NotEmitted) { EmitState = Emitting; - Handle = emitToBaseLayer(BaseLayer); + emitToBaseLayer(BaseLayer); EmitState = Emitted; } - BaseLayer.emitAndFinalize(Handle); + BaseLayer.emitAndFinalize(K); } private: @@ -135,11 +129,11 @@ private: return buildMangledSymbols(Name, ExportedSymbolsOnly); } - Expected<BaseLayerHandleT> emitToBaseLayer(BaseLayerT &BaseLayer) { + Error emitToBaseLayer(BaseLayerT &BaseLayer) { // We don't need the mangled names set any more: Once we've emitted this // to the base layer we'll just look for symbols there. MangledSymbols.reset(); - return BaseLayer.addModule(std::move(M), std::move(Resolver)); + return BaseLayer.addModule(std::move(K), std::move(M)); } // If the mangled name of the given GlobalValue matches the given search @@ -192,46 +186,40 @@ private: } enum { NotEmitted, Emitting, Emitted } EmitState = NotEmitted; - BaseLayerHandleT Handle; - std::shared_ptr<Module> M; - std::shared_ptr<JITSymbolResolver> Resolver; + VModuleKey K; + std::unique_ptr<Module> M; mutable std::unique_ptr<StringMap<const GlobalValue*>> MangledSymbols; }; - using ModuleListT = std::list<std::unique_ptr<EmissionDeferredModule>>; - BaseLayerT &BaseLayer; - ModuleListT ModuleList; + std::map<VModuleKey, std::unique_ptr<EmissionDeferredModule>> ModuleMap; public: - /// @brief Handle to a loaded module. - using ModuleHandleT = typename ModuleListT::iterator; - - /// @brief Construct a lazy emitting layer. + /// Construct a lazy emitting layer. LazyEmittingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} - /// @brief Add the given module to the lazy emitting layer. - Expected<ModuleHandleT> - addModule(std::shared_ptr<Module> M, - std::shared_ptr<JITSymbolResolver> Resolver) { - return ModuleList.insert( - ModuleList.end(), - llvm::make_unique<EmissionDeferredModule>(std::move(M), - std::move(Resolver))); + /// Add the given module to the lazy emitting layer. + Error addModule(VModuleKey K, std::unique_ptr<Module> M) { + assert(!ModuleMap.count(K) && "VModuleKey K already in use"); + ModuleMap[K] = + llvm::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); + return Error::success(); } - /// @brief Remove the module represented by the given handle. + /// Remove the module represented by the given handle. /// /// This method will free the memory associated with the given module, both /// in this layer, and the base layer. - Error removeModule(ModuleHandleT H) { - Error Err = (*H)->removeModuleFromBaseLayer(BaseLayer); - ModuleList.erase(H); - return Err; + Error removeModule(VModuleKey K) { + auto I = ModuleMap.find(K); + assert(I != ModuleMap.end() && "VModuleKey K not valid here"); + auto EDM = std::move(I.second); + ModuleMap.erase(I); + return EDM->removeModuleFromBaseLayer(BaseLayer); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. @@ -243,26 +231,27 @@ public: // If not found then search the deferred modules. If any of these contain a // definition of 'Name' then they will return a JITSymbol that will emit // the corresponding module when the symbol address is requested. - for (auto &DeferredMod : ModuleList) - if (auto Symbol = DeferredMod->find(Name, ExportedSymbolsOnly, BaseLayer)) + for (auto &KV : ModuleMap) + if (auto Symbol = KV.second->find(Name, ExportedSymbolsOnly, BaseLayer)) return Symbol; // If no definition found anywhere return a null symbol. return nullptr; } - /// @brief Get the address of the given symbol in the context of the of - /// compiled modules represented by the handle H. - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + /// Get the address of the given symbol in the context of the of + /// compiled modules represented by the key K. + JITSymbol findSymbolIn(VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return (*H)->find(Name, ExportedSymbolsOnly, BaseLayer); + assert(ModuleMap.count(K) && "VModuleKey K not valid here"); + return ModuleMap[K]->find(Name, ExportedSymbolsOnly, BaseLayer); } - /// @brief Immediately emit and finalize the module represented by the given - /// handle. - /// @param H Handle for module to emit/finalize. - Error emitAndFinalize(ModuleHandleT H) { - return (*H)->emitAndFinalize(BaseLayer); + /// Immediately emit and finalize the module represented by the given + /// key. + Error emitAndFinalize(VModuleKey K) { + assert(ModuleMap.count(K) && "VModuleKey K not valid here"); + return ModuleMap[K]->emitAndFinalize(BaseLayer); } }; diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h new file mode 100644 index 000000000000..52c8c162ff0b --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -0,0 +1,211 @@ +//===--- Legacy.h -- Adapters for ExecutionEngine API interop ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Contains core ORC APIs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LEGACY_H +#define LLVM_EXECUTIONENGINE_ORC_LEGACY_H + +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" + +namespace llvm { +namespace orc { + +/// SymbolResolver is a composable interface for looking up symbol flags +/// and addresses using the AsynchronousSymbolQuery type. It will +/// eventually replace the LegacyJITSymbolResolver interface as the +/// stardard ORC symbol resolver type. +/// +/// FIXME: SymbolResolvers should go away and be replaced with VSOs with +/// defenition generators. +class SymbolResolver { +public: + virtual ~SymbolResolver() = default; + + /// Returns the flags for each symbol in Symbols that can be found, + /// along with the set of symbol that could not be found. + virtual SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) = 0; + + /// For each symbol in Symbols that can be found, assigns that symbols + /// value in Query. Returns the set of symbols that could not be found. + virtual SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) = 0; + +private: + virtual void anchor(); +}; + +/// Implements SymbolResolver with a pair of supplied function objects +/// for convenience. See createSymbolResolver. +template <typename LookupFlagsFn, typename LookupFn> +class LambdaSymbolResolver final : public SymbolResolver { +public: + template <typename LookupFlagsFnRef, typename LookupFnRef> + LambdaSymbolResolver(LookupFlagsFnRef &&LookupFlags, LookupFnRef &&Lookup) + : LookupFlags(std::forward<LookupFlagsFnRef>(LookupFlags)), + Lookup(std::forward<LookupFnRef>(Lookup)) {} + + SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) final { + return LookupFlags(Symbols); + } + + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) final { + return Lookup(std::move(Query), std::move(Symbols)); + } + +private: + LookupFlagsFn LookupFlags; + LookupFn Lookup; +}; + +/// Creates a SymbolResolver implementation from the pair of supplied +/// function objects. +template <typename LookupFlagsFn, typename LookupFn> +std::unique_ptr<LambdaSymbolResolver< + typename std::remove_cv< + typename std::remove_reference<LookupFlagsFn>::type>::type, + typename std::remove_cv< + typename std::remove_reference<LookupFn>::type>::type>> +createSymbolResolver(LookupFlagsFn &&LookupFlags, LookupFn &&Lookup) { + using LambdaSymbolResolverImpl = LambdaSymbolResolver< + typename std::remove_cv< + typename std::remove_reference<LookupFlagsFn>::type>::type, + typename std::remove_cv< + typename std::remove_reference<LookupFn>::type>::type>; + return llvm::make_unique<LambdaSymbolResolverImpl>( + std::forward<LookupFlagsFn>(LookupFlags), std::forward<LookupFn>(Lookup)); +} + +class JITSymbolResolverAdapter : public JITSymbolResolver { +public: + JITSymbolResolverAdapter(ExecutionSession &ES, SymbolResolver &R, + MaterializationResponsibility *MR); + Expected<LookupFlagsResult> lookupFlags(const LookupSet &Symbols) override; + Expected<LookupResult> lookup(const LookupSet &Symbols) override; + +private: + ExecutionSession &ES; + std::set<SymbolStringPtr> ResolvedStrings; + SymbolResolver &R; + MaterializationResponsibility *MR; +}; + +/// Use the given legacy-style FindSymbol function (i.e. a function that +/// takes a const std::string& or StringRef and returns a JITSymbol) to +/// find the flags for each symbol in Symbols and store their flags in +/// SymbolFlags. If any JITSymbol returned by FindSymbol is in an error +/// state the function returns immediately with that error, otherwise it +/// returns the set of symbols not found. +/// +/// Useful for implementing lookupFlags bodies that query legacy resolvers. +template <typename FindSymbolFn> +Expected<SymbolFlagsMap> lookupFlagsWithLegacyFn(const SymbolNameSet &Symbols, + FindSymbolFn FindSymbol) { + SymbolFlagsMap SymbolFlags; + + for (auto &S : Symbols) { + if (JITSymbol Sym = FindSymbol(*S)) + SymbolFlags[S] = Sym.getFlags(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + } + + return SymbolFlags; +} + +/// Use the given legacy-style FindSymbol function (i.e. a function that +/// takes a const std::string& or StringRef and returns a JITSymbol) to +/// find the address and flags for each symbol in Symbols and store the +/// result in Query. If any JITSymbol returned by FindSymbol is in an +/// error then Query.notifyFailed(...) is called with that error and the +/// function returns immediately. On success, returns the set of symbols +/// not found. +/// +/// Useful for implementing lookup bodies that query legacy resolvers. +template <typename FindSymbolFn> +SymbolNameSet +lookupWithLegacyFn(ExecutionSession &ES, AsynchronousSymbolQuery &Query, + const SymbolNameSet &Symbols, FindSymbolFn FindSymbol) { + SymbolNameSet SymbolsNotFound; + bool NewSymbolsResolved = false; + + for (auto &S : Symbols) { + if (JITSymbol Sym = FindSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query.resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + Query.notifySymbolReady(); + NewSymbolsResolved = true; + } else { + ES.legacyFailQuery(Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + ES.legacyFailQuery(Query, std::move(Err)); + return SymbolNameSet(); + } else + SymbolsNotFound.insert(S); + } + + if (NewSymbolsResolved && Query.isFullyResolved()) + Query.handleFullyResolved(); + + if (NewSymbolsResolved && Query.isFullyReady()) + Query.handleFullyReady(); + + return SymbolsNotFound; +} + +/// An ORC SymbolResolver implementation that uses a legacy +/// findSymbol-like function to perform lookup; +template <typename LegacyLookupFn> +class LegacyLookupFnResolver final : public SymbolResolver { +public: + using ErrorReporter = std::function<void(Error)>; + + LegacyLookupFnResolver(ExecutionSession &ES, LegacyLookupFn LegacyLookup, + ErrorReporter ReportError) + : ES(ES), LegacyLookup(std::move(LegacyLookup)), + ReportError(std::move(ReportError)) {} + + SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) final { + if (auto SymbolFlags = lookupFlagsWithLegacyFn(Symbols, LegacyLookup)) + return std::move(*SymbolFlags); + else { + ReportError(SymbolFlags.takeError()); + return SymbolFlagsMap(); + } + } + + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) final { + return lookupWithLegacyFn(ES, *Query, Symbols, LegacyLookup); + } + +private: + ExecutionSession &ES; + LegacyLookupFn LegacyLookup; + ErrorReporter ReportError; +}; + +template <typename LegacyLookupFn> +std::shared_ptr<LegacyLookupFnResolver<LegacyLookupFn>> +createLegacyLookupResolver(ExecutionSession &ES, LegacyLookupFn LegacyLookup, + std::function<void(Error)> ErrorReporter) { + return std::make_shared<LegacyLookupFnResolver<LegacyLookupFn>>( + ES, std::move(LegacyLookup), std::move(ErrorReporter)); +} + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LEGACY_H diff --git a/include/llvm/ExecutionEngine/Orc/NullResolver.h b/include/llvm/ExecutionEngine/Orc/NullResolver.h index 957b94912b3f..3dd3cfe05b8d 100644 --- a/include/llvm/ExecutionEngine/Orc/NullResolver.h +++ b/include/llvm/ExecutionEngine/Orc/NullResolver.h @@ -15,14 +15,23 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_NULLRESOLVER_H #define LLVM_EXECUTIONENGINE_ORC_NULLRESOLVER_H +#include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" namespace llvm { namespace orc { +class NullResolver : public SymbolResolver { +public: + SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) override; + + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) override; +}; + /// SymbolResolver impliementation that rejects all resolution requests. /// Useful for clients that have no cross-object fixups. -class NullResolver : public JITSymbolResolver { +class NullLegacyResolver : public LegacyJITSymbolResolver { public: JITSymbol findSymbol(const std::string &Name) final; diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h index cb47e7520b1a..c6b43a9c8ed6 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -15,6 +15,7 @@ #define LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" #include <algorithm> #include <memory> #include <string> @@ -22,7 +23,24 @@ namespace llvm { namespace orc { -/// @brief Object mutating layer. +class ObjectTransformLayer2 : public ObjectLayer { +public: + using TransformFunction = + std::function<Expected<std::unique_ptr<MemoryBuffer>>( + std::unique_ptr<MemoryBuffer>)>; + + ObjectTransformLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer, + TransformFunction Transform); + + void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) override; + +private: + ObjectLayer &BaseLayer; + TransformFunction Transform; +}; + +/// Object mutating layer. /// /// This layer accepts sets of ObjectFiles (via addObject). It /// immediately applies the user supplied functor to each object, then adds @@ -30,29 +48,24 @@ namespace orc { template <typename BaseLayerT, typename TransformFtor> class ObjectTransformLayer { public: - /// @brief Handle to a set of added objects. - using ObjHandleT = typename BaseLayerT::ObjHandleT; - - /// @brief Construct an ObjectTransformLayer with the given BaseLayer + /// Construct an ObjectTransformLayer with the given BaseLayer ObjectTransformLayer(BaseLayerT &BaseLayer, TransformFtor Transform = TransformFtor()) : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} - /// @brief Apply the transform functor to each object in the object set, then + /// Apply the transform functor to each object in the object set, then /// add the resulting set of objects to the base layer, along with the /// memory manager and symbol resolver. /// /// @return A handle for the added objects. - template <typename ObjectPtr> - Expected<ObjHandleT> addObject(ObjectPtr Obj, - std::shared_ptr<JITSymbolResolver> Resolver) { - return BaseLayer.addObject(Transform(std::move(Obj)), std::move(Resolver)); + template <typename ObjectPtr> Error addObject(VModuleKey K, ObjectPtr Obj) { + return BaseLayer.addObject(std::move(K), Transform(std::move(Obj))); } - /// @brief Remove the object set associated with the handle H. - Error removeObject(ObjHandleT H) { return BaseLayer.removeObject(H); } + /// Remove the object set associated with the VModuleKey K. + Error removeObject(VModuleKey K) { return BaseLayer.removeObject(K); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. @@ -60,36 +73,34 @@ public: return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } - /// @brief Get the address of the given symbol in the context of the set of - /// objects represented by the handle H. This call is forwarded to the - /// base layer's implementation. - /// @param H The handle for the object set to search in. + /// Get the address of the given symbol in the context of the set of + /// objects represented by the VModuleKey K. This call is forwarded to + /// the base layer's implementation. + /// @param K The VModuleKey associated with the object set to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the /// given object set. - JITSymbol findSymbolIn(ObjHandleT H, const std::string &Name, + JITSymbol findSymbolIn(VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); + return BaseLayer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - /// @brief Immediately emit and finalize the object set represented by the - /// given handle. - /// @param H Handle for object set to emit/finalize. - Error emitAndFinalize(ObjHandleT H) { - return BaseLayer.emitAndFinalize(H); - } + /// Immediately emit and finalize the object set represented by the + /// given VModuleKey K. + Error emitAndFinalize(VModuleKey K) { return BaseLayer.emitAndFinalize(K); } - /// @brief Map section addresses for the objects associated with the handle H. - void mapSectionAddress(ObjHandleT H, const void *LocalAddress, + /// Map section addresses for the objects associated with the + /// VModuleKey K. + void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) { - BaseLayer.mapSectionAddress(H, LocalAddress, TargetAddr); + BaseLayer.mapSectionAddress(K, LocalAddress, TargetAddr); } - /// @brief Access the transform functor directly. + /// Access the transform functor directly. TransformFtor &getTransform() { return Transform; } - /// @brief Access the mumate functor directly. + /// Access the mumate functor directly. const TransformFtor &getTransform() const { return Transform; } private: diff --git a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h index e1b55649b9f2..581c598aff62 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -71,7 +71,7 @@ public: } }; -/// @brief Provide information about stub blocks generated by the +/// Provide information about stub blocks generated by the /// makeIndirectStubsBlock function. template <unsigned StubSizeVal> class GenericIndirectStubsInfo { public: @@ -92,16 +92,16 @@ public: return *this; } - /// @brief Number of stubs in this block. + /// Number of stubs in this block. unsigned getNumStubs() const { return NumStubs; } - /// @brief Get a pointer to the stub at the given index, which must be in + /// Get a pointer to the stub at the given index, which must be in /// the range 0 .. getNumStubs() - 1. void *getStub(unsigned Idx) const { return static_cast<char *>(StubsMem.base()) + Idx * StubSize; } - /// @brief Get a pointer to the implementation-pointer at the given index, + /// Get a pointer to the implementation-pointer at the given index, /// which must be in the range 0 .. getNumStubs() - 1. void **getPtr(unsigned Idx) const { char *PtrsBase = static_cast<char *>(StubsMem.base()) + NumStubs * StubSize; @@ -124,18 +124,18 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the resolver code into the given memory. The user is be + /// Write the resolver code into the given memory. The user is be /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); - /// @brief Write the requsted number of trampolines into the given memory, + /// Write the requsted number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); - /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to + /// Emit at least MinStubs worth of indirect call stubs, rounded out to /// the nearest page size. /// /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k @@ -145,7 +145,7 @@ public: unsigned MinStubs, void *InitialPtrVal); }; -/// @brief X86_64 code that's common to all ABIs. +/// X86_64 code that's common to all ABIs. /// /// X86_64 supports lazy JITing. class OrcX86_64_Base { @@ -155,13 +155,13 @@ public: using IndirectStubsInfo = GenericIndirectStubsInfo<8>; - /// @brief Write the requsted number of trampolines into the given memory, + /// Write the requsted number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); - /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to + /// Emit at least MinStubs worth of indirect call stubs, rounded out to /// the nearest page size. /// /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k @@ -171,7 +171,7 @@ public: unsigned MinStubs, void *InitialPtrVal); }; -/// @brief X86_64 support for SysV ABI (Linux, MacOSX). +/// X86_64 support for SysV ABI (Linux, MacOSX). /// /// X86_64_SysV supports lazy JITing. class OrcX86_64_SysV : public OrcX86_64_Base { @@ -181,13 +181,13 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the resolver code into the given memory. The user is be + /// Write the resolver code into the given memory. The user is be /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); }; -/// @brief X86_64 support for Win32. +/// X86_64 support for Win32. /// /// X86_64_Win32 supports lazy JITing. class OrcX86_64_Win32 : public OrcX86_64_Base { @@ -197,13 +197,13 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the resolver code into the given memory. The user is be + /// Write the resolver code into the given memory. The user is be /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); }; -/// @brief I386 support. +/// I386 support. /// /// I386 supports lazy JITing. class OrcI386 { @@ -217,18 +217,18 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the resolver code into the given memory. The user is be + /// Write the resolver code into the given memory. The user is be /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); - /// @brief Write the requsted number of trampolines into the given memory, + /// Write the requsted number of trampolines into the given memory, /// which must be big enough to hold 1 pointer, plus NumTrampolines /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); - /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to + /// Emit at least MinStubs worth of indirect call stubs, rounded out to /// the nearest page size. /// /// E.g. Asking for 4 stubs on i386, where stubs are 8-bytes, with 4k diff --git a/include/llvm/ExecutionEngine/Orc/OrcError.h b/include/llvm/ExecutionEngine/Orc/OrcError.h index e1ac87075ac0..dc60e8d74e97 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -22,7 +22,9 @@ namespace orc { enum class OrcErrorCode : int { // RPC Errors - JITSymbolNotFound = 1, + UnknownORCError = 1, + DuplicateDefinition, + JITSymbolNotFound, RemoteAllocatorDoesNotExist, RemoteAllocatorIdAlreadyInUse, RemoteMProtectAddrUnrecognized, @@ -39,6 +41,18 @@ enum class OrcErrorCode : int { std::error_code orcError(OrcErrorCode ErrCode); +class DuplicateDefinition : public ErrorInfo<DuplicateDefinition> { +public: + static char ID; + + DuplicateDefinition(std::string SymbolName); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getSymbolName() const; +private: + std::string SymbolName; +}; + class JITSymbolNotFound : public ErrorInfo<JITSymbolNotFound> { public: static char ID; diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 7179e5ff66fd..739e5ba47c12 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -63,7 +63,7 @@ public: public: ~RemoteRTDyldMemoryManager() { Client.destroyRemoteAllocator(Id); - DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); + LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); } RemoteRTDyldMemoryManager(const RemoteRTDyldMemoryManager &) = delete; @@ -79,9 +79,9 @@ public: Unmapped.back().CodeAllocs.emplace_back(Size, Alignment); uint8_t *Alloc = reinterpret_cast<uint8_t *>( Unmapped.back().CodeAllocs.back().getLocalAddress()); - DEBUG(dbgs() << "Allocator " << Id << " allocated code for " - << SectionName << ": " << Alloc << " (" << Size - << " bytes, alignment " << Alignment << ")\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated code for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); return Alloc; } @@ -92,18 +92,18 @@ public: Unmapped.back().RODataAllocs.emplace_back(Size, Alignment); uint8_t *Alloc = reinterpret_cast<uint8_t *>( Unmapped.back().RODataAllocs.back().getLocalAddress()); - DEBUG(dbgs() << "Allocator " << Id << " allocated ro-data for " - << SectionName << ": " << Alloc << " (" << Size - << " bytes, alignment " << Alignment << ")\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated ro-data for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); return Alloc; } // else... Unmapped.back().RWDataAllocs.emplace_back(Size, Alignment); uint8_t *Alloc = reinterpret_cast<uint8_t *>( Unmapped.back().RWDataAllocs.back().getLocalAddress()); - DEBUG(dbgs() << "Allocator " << Id << " allocated rw-data for " - << SectionName << ": " << Alloc << " (" << Size - << " bytes, alignment " << Alignment << ")\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated rw-data for " + << SectionName << ": " << Alloc << " (" << Size + << " bytes, alignment " << Alignment << ")\n"); return Alloc; } @@ -113,36 +113,36 @@ public: uint32_t RWDataAlign) override { Unmapped.push_back(ObjectAllocs()); - DEBUG(dbgs() << "Allocator " << Id << " reserved:\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " reserved:\n"); if (CodeSize != 0) { Unmapped.back().RemoteCodeAddr = Client.reserveMem(Id, CodeSize, CodeAlign); - DEBUG(dbgs() << " code: " - << format("0x%016x", Unmapped.back().RemoteCodeAddr) - << " (" << CodeSize << " bytes, alignment " << CodeAlign - << ")\n"); + LLVM_DEBUG(dbgs() << " code: " + << format("0x%016x", Unmapped.back().RemoteCodeAddr) + << " (" << CodeSize << " bytes, alignment " + << CodeAlign << ")\n"); } if (RODataSize != 0) { Unmapped.back().RemoteRODataAddr = Client.reserveMem(Id, RODataSize, RODataAlign); - DEBUG(dbgs() << " ro-data: " - << format("0x%016x", Unmapped.back().RemoteRODataAddr) - << " (" << RODataSize << " bytes, alignment " - << RODataAlign << ")\n"); + LLVM_DEBUG(dbgs() << " ro-data: " + << format("0x%016x", Unmapped.back().RemoteRODataAddr) + << " (" << RODataSize << " bytes, alignment " + << RODataAlign << ")\n"); } if (RWDataSize != 0) { Unmapped.back().RemoteRWDataAddr = Client.reserveMem(Id, RWDataSize, RWDataAlign); - DEBUG(dbgs() << " rw-data: " - << format("0x%016x", Unmapped.back().RemoteRWDataAddr) - << " (" << RWDataSize << " bytes, alignment " - << RWDataAlign << ")\n"); + LLVM_DEBUG(dbgs() << " rw-data: " + << format("0x%016x", Unmapped.back().RemoteRWDataAddr) + << " (" << RWDataSize << " bytes, alignment " + << RWDataAlign << ")\n"); } } @@ -162,7 +162,7 @@ public: void notifyObjectLoaded(RuntimeDyld &Dyld, const object::ObjectFile &Obj) override { - DEBUG(dbgs() << "Allocator " << Id << " applied mappings:\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " applied mappings:\n"); for (auto &ObjAllocs : Unmapped) { mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, ObjAllocs.RemoteCodeAddr); @@ -176,7 +176,7 @@ public: } bool finalizeMemory(std::string *ErrMsg = nullptr) override { - DEBUG(dbgs() << "Allocator " << Id << " finalizing:\n"); + LLVM_DEBUG(dbgs() << "Allocator " << Id << " finalizing:\n"); for (auto &ObjAllocs : Unfinalized) { if (copyAndProtect(ObjAllocs.CodeAllocs, ObjAllocs.RemoteCodeAddr, @@ -261,7 +261,7 @@ public: RemoteRTDyldMemoryManager(OrcRemoteTargetClient &Client, ResourceIdMgr::ResourceId Id) : Client(Client), Id(Id) { - DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); + LLVM_DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); } // Maps all allocations in Allocs to aligned blocks @@ -270,8 +270,9 @@ public: for (auto &Alloc : Allocs) { NextAddr = alignTo(NextAddr, Alloc.getAlign()); Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextAddr); - DEBUG(dbgs() << " " << static_cast<void *>(Alloc.getLocalAddress()) - << " -> " << format("0x%016x", NextAddr) << "\n"); + LLVM_DEBUG(dbgs() << " " + << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " << format("0x%016x", NextAddr) << "\n"); Alloc.setRemoteAddress(NextAddr); // Only advance NextAddr if it was non-null to begin with, @@ -290,22 +291,23 @@ public: assert(!Allocs.empty() && "No sections in allocated segment"); for (auto &Alloc : Allocs) { - DEBUG(dbgs() << " copying section: " - << static_cast<void *>(Alloc.getLocalAddress()) << " -> " - << format("0x%016x", Alloc.getRemoteAddress()) << " (" - << Alloc.getSize() << " bytes)\n";); + LLVM_DEBUG(dbgs() << " copying section: " + << static_cast<void *>(Alloc.getLocalAddress()) + << " -> " + << format("0x%016x", Alloc.getRemoteAddress()) + << " (" << Alloc.getSize() << " bytes)\n";); if (Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), Alloc.getSize())) return true; } - DEBUG(dbgs() << " setting " - << (Permissions & sys::Memory::MF_READ ? 'R' : '-') - << (Permissions & sys::Memory::MF_WRITE ? 'W' : '-') - << (Permissions & sys::Memory::MF_EXEC ? 'X' : '-') - << " permissions on block: " - << format("0x%016x", RemoteSegmentAddr) << "\n"); + LLVM_DEBUG(dbgs() << " setting " + << (Permissions & sys::Memory::MF_READ ? 'R' : '-') + << (Permissions & sys::Memory::MF_WRITE ? 'W' : '-') + << (Permissions & sys::Memory::MF_EXEC ? 'X' : '-') + << " permissions on block: " + << format("0x%016x", RemoteSegmentAddr) << "\n"); if (Client.setProtections(Id, RemoteSegmentAddr, Permissions)) return true; } @@ -356,25 +358,25 @@ public: return Error::success(); } - JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { auto I = StubIndexes.find(Name); if (I == StubIndexes.end()) return nullptr; auto Key = I->second.first; auto Flags = I->second.second; - auto StubSymbol = JITSymbol(getStubAddr(Key), Flags); + auto StubSymbol = JITEvaluatedSymbol(getStubAddr(Key), Flags); if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) return nullptr; return StubSymbol; } - JITSymbol findPointer(StringRef Name) override { + JITEvaluatedSymbol findPointer(StringRef Name) override { auto I = StubIndexes.find(Name); if (I == StubIndexes.end()) return nullptr; auto Key = I->second.first; auto Flags = I->second.second; - return JITSymbol(getPtrAddr(Key), Flags); + return JITEvaluatedSymbol(getPtrAddr(Key), Flags); } Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { @@ -449,8 +451,9 @@ public: class RemoteCompileCallbackManager : public JITCompileCallbackManager { public: RemoteCompileCallbackManager(OrcRemoteTargetClient &Client, + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) - : JITCompileCallbackManager(ErrorHandlerAddress), Client(Client) {} + : JITCompileCallbackManager(ES, ErrorHandlerAddress), Client(Client) {} private: Error grow() override { @@ -475,10 +478,10 @@ public: /// Channel is the ChannelT instance to communicate on. It is assumed that /// the channel is ready to be read from and written to. static Expected<std::unique_ptr<OrcRemoteTargetClient>> - Create(rpc::RawByteChannel &Channel, std::function<void(Error)> ReportError) { + Create(rpc::RawByteChannel &Channel, ExecutionSession &ES) { Error Err = Error::success(); auto Client = std::unique_ptr<OrcRemoteTargetClient>( - new OrcRemoteTargetClient(Channel, std::move(ReportError), Err)); + new OrcRemoteTargetClient(Channel, ES, Err)); if (Err) return std::move(Err); return std::move(Client); @@ -487,7 +490,8 @@ public: /// Call the int(void) function at the given address in the target and return /// its result. Expected<int> callIntVoid(JITTargetAddress Addr) { - DEBUG(dbgs() << "Calling int(*)(void) " << format("0x%016x", Addr) << "\n"); + LLVM_DEBUG(dbgs() << "Calling int(*)(void) " << format("0x%016x", Addr) + << "\n"); return callB<exec::CallIntVoid>(Addr); } @@ -495,16 +499,16 @@ public: /// return its result. Expected<int> callMain(JITTargetAddress Addr, const std::vector<std::string> &Args) { - DEBUG(dbgs() << "Calling int(*)(int, char*[]) " << format("0x%016x", Addr) - << "\n"); + LLVM_DEBUG(dbgs() << "Calling int(*)(int, char*[]) " + << format("0x%016x", Addr) << "\n"); return callB<exec::CallMain>(Addr, Args); } /// Call the void() function at the given address in the target and wait for /// it to finish. Error callVoidVoid(JITTargetAddress Addr) { - DEBUG(dbgs() << "Calling void(*)(void) " << format("0x%016x", Addr) - << "\n"); + LLVM_DEBUG(dbgs() << "Calling void(*)(void) " << format("0x%016x", Addr) + << "\n"); return callB<exec::CallVoidVoid>(Addr); } @@ -531,12 +535,14 @@ public: Expected<RemoteCompileCallbackManager &> enableCompileCallbacks(JITTargetAddress ErrorHandlerAddress) { + assert(!CallbackManager && "CallbackManager already obtained"); + // Emit the resolver block on the JIT server. if (auto Err = callB<stubs::EmitResolverBlock>()) return std::move(Err); // Create the callback manager. - CallbackManager.emplace(*this, ErrorHandlerAddress); + CallbackManager.emplace(*this, ES, ErrorHandlerAddress); RemoteCompileCallbackManager &Mgr = *CallbackManager; return Mgr; } @@ -554,10 +560,10 @@ public: Error terminateSession() { return callB<utils::TerminateSession>(); } private: - OrcRemoteTargetClient(rpc::RawByteChannel &Channel, - std::function<void(Error)> ReportError, Error &Err) + OrcRemoteTargetClient(rpc::RawByteChannel &Channel, ExecutionSession &ES, + Error &Err) : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(Channel, true), - ReportError(std::move(ReportError)) { + ES(ES) { ErrorAsOutParameter EAO(&Err); addHandler<utils::RequestCompile>( @@ -577,7 +583,7 @@ private: void deregisterEHFrames(JITTargetAddress Addr, uint32_t Size) { if (auto Err = callB<eh::RegisterEHFrames>(Addr, Size)) - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); } void destroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { @@ -592,7 +598,7 @@ private: void destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { IndirectStubOwnerIds.release(Id); if (auto Err = callB<stubs::DestroyIndirectStubsOwner>(Id)) - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); } Expected<std::tuple<JITTargetAddress, JITTargetAddress, uint32_t>> @@ -625,7 +631,7 @@ private: if (auto AddrOrErr = callB<mem::ReserveMem>(Id, Size, Align)) return *AddrOrErr; else { - ReportError(AddrOrErr.takeError()); + ES.reportError(AddrOrErr.takeError()); return 0; } } @@ -633,7 +639,7 @@ private: bool setProtections(ResourceIdMgr::ResourceId Id, JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { if (auto Err = callB<mem::SetProtections>(Id, RemoteSegAddr, ProtFlags)) { - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); return true; } else return false; @@ -641,7 +647,7 @@ private: bool writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { if (auto Err = callB<mem::WriteMem>(DirectBufferWriter(Src, Addr, Size))) { - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); return true; } else return false; @@ -653,6 +659,7 @@ private: static Error doNothing() { return Error::success(); } + ExecutionSession &ES; std::function<void(Error)> ReportError; std::string RemoteTargetTriple; uint32_t RemotePointerSize = 0; diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h index cf419d33004c..acbc1682fa5d 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -161,9 +161,9 @@ private: IntVoidFnTy Fn = reinterpret_cast<IntVoidFnTy>(static_cast<uintptr_t>(Addr)); - DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); + LLVM_DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); int Result = Fn(); - DEBUG(dbgs() << " Result = " << Result << "\n"); + LLVM_DEBUG(dbgs() << " Result = " << Result << "\n"); return Result; } @@ -180,15 +180,13 @@ private: for (auto &Arg : Args) ArgV[Idx++] = Arg.c_str(); ArgV[ArgC] = 0; - DEBUG( - for (int Idx = 0; Idx < ArgC; ++Idx) { - llvm::dbgs() << "Arg " << Idx << ": " << ArgV[Idx] << "\n"; - } - ); + LLVM_DEBUG(for (int Idx = 0; Idx < ArgC; ++Idx) { + llvm::dbgs() << "Arg " << Idx << ": " << ArgV[Idx] << "\n"; + }); - DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); + LLVM_DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); int Result = Fn(ArgC, ArgV.get()); - DEBUG(dbgs() << " Result = " << Result << "\n"); + LLVM_DEBUG(dbgs() << " Result = " << Result << "\n"); return Result; } @@ -199,9 +197,9 @@ private: VoidVoidFnTy Fn = reinterpret_cast<VoidVoidFnTy>(static_cast<uintptr_t>(Addr)); - DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); + LLVM_DEBUG(dbgs() << " Calling " << format("0x%016x", Addr) << "\n"); Fn(); - DEBUG(dbgs() << " Complete.\n"); + LLVM_DEBUG(dbgs() << " Complete.\n"); return Error::success(); } @@ -211,7 +209,7 @@ private: if (I != Allocators.end()) return errorCodeToError( orcError(OrcErrorCode::RemoteAllocatorIdAlreadyInUse)); - DEBUG(dbgs() << " Created allocator " << Id << "\n"); + LLVM_DEBUG(dbgs() << " Created allocator " << Id << "\n"); Allocators[Id] = Allocator(); return Error::success(); } @@ -221,15 +219,16 @@ private: if (I != IndirectStubsOwners.end()) return errorCodeToError( orcError(OrcErrorCode::RemoteIndirectStubsOwnerIdAlreadyInUse)); - DEBUG(dbgs() << " Create indirect stubs owner " << Id << "\n"); + LLVM_DEBUG(dbgs() << " Create indirect stubs owner " << Id << "\n"); IndirectStubsOwners[Id] = ISBlockOwnerList(); return Error::success(); } Error handleDeregisterEHFrames(JITTargetAddress TAddr, uint32_t Size) { uint8_t *Addr = reinterpret_cast<uint8_t *>(static_cast<uintptr_t>(TAddr)); - DEBUG(dbgs() << " Registering EH frames at " << format("0x%016x", TAddr) - << ", Size = " << Size << " bytes\n"); + LLVM_DEBUG(dbgs() << " Registering EH frames at " + << format("0x%016x", TAddr) << ", Size = " << Size + << " bytes\n"); EHFramesDeregister(Addr, Size); return Error::success(); } @@ -240,7 +239,7 @@ private: return errorCodeToError( orcError(OrcErrorCode::RemoteAllocatorDoesNotExist)); Allocators.erase(I); - DEBUG(dbgs() << " Destroyed allocator " << Id << "\n"); + LLVM_DEBUG(dbgs() << " Destroyed allocator " << Id << "\n"); return Error::success(); } @@ -256,8 +255,8 @@ private: Expected<std::tuple<JITTargetAddress, JITTargetAddress, uint32_t>> handleEmitIndirectStubs(ResourceIdMgr::ResourceId Id, uint32_t NumStubsRequired) { - DEBUG(dbgs() << " ISMgr " << Id << " request " << NumStubsRequired - << " stubs.\n"); + LLVM_DEBUG(dbgs() << " ISMgr " << Id << " request " << NumStubsRequired + << " stubs.\n"); auto StubOwnerItr = IndirectStubsOwners.find(Id); if (StubOwnerItr == IndirectStubsOwners.end()) @@ -328,8 +327,8 @@ private: Expected<JITTargetAddress> handleGetSymbolAddress(const std::string &Name) { JITTargetAddress Addr = SymbolLookup(Name); - DEBUG(dbgs() << " Symbol '" << Name << "' = " << format("0x%016x", Addr) - << "\n"); + LLVM_DEBUG(dbgs() << " Symbol '" << Name + << "' = " << format("0x%016x", Addr) << "\n"); return Addr; } @@ -340,12 +339,13 @@ private: uint32_t PageSize = sys::Process::getPageSize(); uint32_t TrampolineSize = TargetT::TrampolineSize; uint32_t IndirectStubSize = TargetT::IndirectStubsInfo::StubSize; - DEBUG(dbgs() << " Remote info:\n" - << " triple = '" << ProcessTriple << "'\n" - << " pointer size = " << PointerSize << "\n" - << " page size = " << PageSize << "\n" - << " trampoline size = " << TrampolineSize << "\n" - << " indirect stub size = " << IndirectStubSize << "\n"); + LLVM_DEBUG(dbgs() << " Remote info:\n" + << " triple = '" << ProcessTriple << "'\n" + << " pointer size = " << PointerSize << "\n" + << " page size = " << PageSize << "\n" + << " trampoline size = " << TrampolineSize << "\n" + << " indirect stub size = " << IndirectStubSize + << "\n"); return std::make_tuple(ProcessTriple, PointerSize, PageSize, TrampolineSize, IndirectStubSize); } @@ -354,8 +354,8 @@ private: uint64_t Size) { uint8_t *Src = reinterpret_cast<uint8_t *>(static_cast<uintptr_t>(RSrc)); - DEBUG(dbgs() << " Reading " << Size << " bytes from " - << format("0x%016x", RSrc) << "\n"); + LLVM_DEBUG(dbgs() << " Reading " << Size << " bytes from " + << format("0x%016x", RSrc) << "\n"); std::vector<uint8_t> Buffer; Buffer.resize(Size); @@ -367,8 +367,9 @@ private: Error handleRegisterEHFrames(JITTargetAddress TAddr, uint32_t Size) { uint8_t *Addr = reinterpret_cast<uint8_t *>(static_cast<uintptr_t>(TAddr)); - DEBUG(dbgs() << " Registering EH frames at " << format("0x%016x", TAddr) - << ", Size = " << Size << " bytes\n"); + LLVM_DEBUG(dbgs() << " Registering EH frames at " + << format("0x%016x", TAddr) << ", Size = " << Size + << " bytes\n"); EHFramesRegister(Addr, Size); return Error::success(); } @@ -384,8 +385,9 @@ private: if (auto Err = Allocator.allocate(LocalAllocAddr, Size, Align)) return std::move(Err); - DEBUG(dbgs() << " Allocator " << Id << " reserved " << LocalAllocAddr - << " (" << Size << " bytes, alignment " << Align << ")\n"); + LLVM_DEBUG(dbgs() << " Allocator " << Id << " reserved " << LocalAllocAddr + << " (" << Size << " bytes, alignment " << Align + << ")\n"); JITTargetAddress AllocAddr = static_cast<JITTargetAddress>( reinterpret_cast<uintptr_t>(LocalAllocAddr)); @@ -401,10 +403,11 @@ private: orcError(OrcErrorCode::RemoteAllocatorDoesNotExist)); auto &Allocator = I->second; void *LocalAddr = reinterpret_cast<void *>(static_cast<uintptr_t>(Addr)); - DEBUG(dbgs() << " Allocator " << Id << " set permissions on " << LocalAddr - << " to " << (Flags & sys::Memory::MF_READ ? 'R' : '-') - << (Flags & sys::Memory::MF_WRITE ? 'W' : '-') - << (Flags & sys::Memory::MF_EXEC ? 'X' : '-') << "\n"); + LLVM_DEBUG(dbgs() << " Allocator " << Id << " set permissions on " + << LocalAddr << " to " + << (Flags & sys::Memory::MF_READ ? 'R' : '-') + << (Flags & sys::Memory::MF_WRITE ? 'W' : '-') + << (Flags & sys::Memory::MF_EXEC ? 'X' : '-') << "\n"); return Allocator.setProtections(LocalAddr, Flags); } @@ -414,14 +417,14 @@ private: } Error handleWriteMem(DirectBufferWriter DBW) { - DEBUG(dbgs() << " Writing " << DBW.getSize() << " bytes to " - << format("0x%016x", DBW.getDst()) << "\n"); + LLVM_DEBUG(dbgs() << " Writing " << DBW.getSize() << " bytes to " + << format("0x%016x", DBW.getDst()) << "\n"); return Error::success(); } Error handleWritePtr(JITTargetAddress Addr, JITTargetAddress PtrVal) { - DEBUG(dbgs() << " Writing pointer *" << format("0x%016x", Addr) << " = " - << format("0x%016x", PtrVal) << "\n"); + LLVM_DEBUG(dbgs() << " Writing pointer *" << format("0x%016x", Addr) + << " = " << format("0x%016x", PtrVal) << "\n"); uintptr_t *Ptr = reinterpret_cast<uintptr_t *>(static_cast<uintptr_t>(Addr)); *Ptr = static_cast<uintptr_t>(PtrVal); diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h index c278cb176853..47bd90bb1bad 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -1631,7 +1631,7 @@ RPCAsyncDispatch<RPCEndpointT, Func> rpcAsyncDispatch(RPCEndpointT &Endpoint) { return RPCAsyncDispatch<RPCEndpointT, Func>(Endpoint); } -/// \brief Allows a set of asynchrounous calls to be dispatched, and then +/// Allows a set of asynchrounous calls to be dispatched, and then /// waited on as a group. class ParallelCallGroup { public: @@ -1640,7 +1640,7 @@ public: ParallelCallGroup(const ParallelCallGroup &) = delete; ParallelCallGroup &operator=(const ParallelCallGroup &) = delete; - /// \brief Make as asynchronous call. + /// Make as asynchronous call. template <typename AsyncDispatcher, typename HandlerT, typename... ArgTs> Error call(const AsyncDispatcher &AsyncDispatch, HandlerT Handler, const ArgTs &... Args) { @@ -1669,7 +1669,7 @@ public: return AsyncDispatch(std::move(WrappedHandler), Args...); } - /// \brief Blocks until all calls have been completed and their return value + /// Blocks until all calls have been completed and their return value /// handlers run. void wait() { std::unique_lock<std::mutex> Lock(M); @@ -1683,21 +1683,21 @@ private: uint32_t NumOutstandingCalls = 0; }; -/// @brief Convenience class for grouping RPC Functions into APIs that can be +/// Convenience class for grouping RPC Functions into APIs that can be /// negotiated as a block. /// template <typename... Funcs> class APICalls { public: - /// @brief Test whether this API contains Function F. + /// Test whether this API contains Function F. template <typename F> class Contains { public: static const bool value = false; }; - /// @brief Negotiate all functions in this API. + /// Negotiate all functions in this API. template <typename RPCEndpoint> static Error negotiate(RPCEndpoint &R) { return Error::success(); diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 246c57341f35..48b3f7a58ed7 100644 --- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -18,6 +18,9 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Error.h" @@ -33,15 +36,62 @@ namespace llvm { namespace orc { -class RTDyldObjectLinkingLayerBase { +class RTDyldObjectLinkingLayer2 : public ObjectLayer { public: + /// Functor for receiving object-loaded notifications. + using NotifyLoadedFunction = + std::function<void(VModuleKey, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &)>; + + /// Functor for receiving finalization notifications. + using NotifyFinalizedFunction = std::function<void(VModuleKey)>; - using ObjectPtr = - std::shared_ptr<object::OwningBinary<object::ObjectFile>>; + using GetMemoryManagerFunction = + std::function<std::shared_ptr<RuntimeDyld::MemoryManager>(VModuleKey)>; + + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, + /// and NotifyFinalized functors. + RTDyldObjectLinkingLayer2( + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, + NotifyLoadedFunction NotifyLoaded = NotifyLoadedFunction(), + NotifyFinalizedFunction NotifyFinalized = NotifyFinalizedFunction()); + + /// Emit the object. + void emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) override; + + /// Map section addresses for the object associated with the + /// VModuleKey K. + void mapSectionAddress(VModuleKey K, const void *LocalAddress, + JITTargetAddress TargetAddr) const; + + /// Set the 'ProcessAllSections' flag. + /// + /// If set to true, all sections in each object file will be allocated using + /// the memory manager, rather than just the sections required for execution. + /// + /// This is kludgy, and may be removed in the future. + void setProcessAllSections(bool ProcessAllSections) { + this->ProcessAllSections = ProcessAllSections; + } + +private: + mutable std::mutex RTDyldLayerMutex; + GetMemoryManagerFunction GetMemoryManager; + NotifyLoadedFunction NotifyLoaded; + NotifyFinalizedFunction NotifyFinalized; + bool ProcessAllSections; + std::map<VModuleKey, RuntimeDyld *> ActiveRTDylds; + std::map<VModuleKey, std::shared_ptr<RuntimeDyld::MemoryManager>> MemMgrs; +}; + +class RTDyldObjectLinkingLayerBase { +public: + using ObjectPtr = std::unique_ptr<MemoryBuffer>; protected: - /// @brief Holds an object to be allocated/linked as a unit in the JIT. + /// Holds an object to be allocated/linked as a unit in the JIT. /// /// An instance of this class will be created for each object added /// via JITObjectLayer::addObject. Deleting the instance (via @@ -55,7 +105,7 @@ protected: void operator=(const LinkedObject&) = delete; virtual ~LinkedObject() = default; - virtual void finalize() = 0; + virtual Error finalize() = 0; virtual JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) = 0; @@ -79,15 +129,9 @@ protected: StringMap<JITEvaluatedSymbol> SymbolTable; bool Finalized = false; }; - - using LinkedObjectListT = std::list<std::unique_ptr<LinkedObject>>; - -public: - /// @brief Handle to a loaded object. - using ObjHandleT = LinkedObjectListT::iterator; }; -/// @brief Bare bones object linking layer. +/// Bare bones object linking layer. /// /// This class is intended to be used as the base layer for a JIT. It allows /// object files to be loaded into memory, linked, and the addresses of their @@ -98,67 +142,93 @@ public: using RTDyldObjectLinkingLayerBase::ObjectPtr; - /// @brief Functor for receiving object-loaded notifications. + /// Functor for receiving object-loaded notifications. using NotifyLoadedFtor = - std::function<void(ObjHandleT, const ObjectPtr &Obj, - const RuntimeDyld::LoadedObjectInfo &)>; + std::function<void(VModuleKey, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &)>; - /// @brief Functor for receiving finalization notifications. - using NotifyFinalizedFtor = std::function<void(ObjHandleT)>; + /// Functor for receiving finalization notifications. + using NotifyFinalizedFtor = + std::function<void(VModuleKey, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &)>; -private: + /// Functor for receiving deallocation notifications. + using NotifyFreedFtor = std::function<void(VModuleKey, const object::ObjectFile &Obj)>; +private: + using OwnedObject = object::OwningBinary<object::ObjectFile>; - template <typename MemoryManagerPtrT, typename SymbolResolverPtrT, - typename FinalizerFtor> + template <typename MemoryManagerPtrT> class ConcreteLinkedObject : public LinkedObject { public: - ConcreteLinkedObject(ObjectPtr Obj, MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver, - FinalizerFtor Finalizer, + ConcreteLinkedObject(RTDyldObjectLinkingLayer &Parent, VModuleKey K, + OwnedObject Obj, MemoryManagerPtrT MemMgr, + std::shared_ptr<SymbolResolver> Resolver, bool ProcessAllSections) - : MemMgr(std::move(MemMgr)), - PFC(llvm::make_unique<PreFinalizeContents>(std::move(Obj), - std::move(Resolver), - std::move(Finalizer), - ProcessAllSections)) { + : K(std::move(K)), + Parent(Parent), + MemMgr(std::move(MemMgr)), + PFC(llvm::make_unique<PreFinalizeContents>( + std::move(Obj), std::move(Resolver), + ProcessAllSections)) { buildInitialSymbolTable(PFC->Obj); } ~ConcreteLinkedObject() override { - MemMgr->deregisterEHFrames(); - } + if (this->Parent.NotifyFreed) + this->Parent.NotifyFreed(K, *ObjForNotify.getBinary()); - void setHandle(ObjHandleT H) { - PFC->Handle = H; + MemMgr->deregisterEHFrames(); } - void finalize() override { + Error finalize() override { assert(PFC && "mapSectionAddress called on finalized LinkedObject"); - RuntimeDyld RTDyld(*MemMgr, *PFC->Resolver); - RTDyld.setProcessAllSections(PFC->ProcessAllSections); - PFC->RTDyld = &RTDyld; + JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver, + nullptr); + PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); + PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections); + + Finalized = true; + + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info = + PFC->RTDyld->loadObject(*PFC->Obj.getBinary()); + + // Copy the symbol table out of the RuntimeDyld instance. + { + auto SymTab = PFC->RTDyld->getSymbolTable(); + for (auto &KV : SymTab) + SymbolTable[KV.first] = KV.second; + } + + if (Parent.NotifyLoaded) + Parent.NotifyLoaded(K, *PFC->Obj.getBinary(), *Info); + + PFC->RTDyld->finalizeWithMemoryManagerLocking(); + + if (PFC->RTDyld->hasError()) + return make_error<StringError>(PFC->RTDyld->getErrorString(), + inconvertibleErrorCode()); - this->Finalized = true; - PFC->Finalizer(PFC->Handle, RTDyld, std::move(PFC->Obj), - [&]() { - this->updateSymbolTable(RTDyld); - }); + if (Parent.NotifyFinalized) + Parent.NotifyFinalized(K, *PFC->Obj.getBinary(), *Info); // Release resources. + if (this->Parent.NotifyFreed) + ObjForNotify = std::move(PFC->Obj); // needed for callback PFC = nullptr; + return Error::success(); } JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) override { - return - [this, Name]() { - // The symbol may be materialized between the creation of this lambda - // and its execution, so we need to double check. - if (!this->Finalized) - this->finalize(); - return this->getSymbol(Name, false).getAddress(); - }; + return [this, Name]() -> Expected<JITTargetAddress> { + // The symbol may be materialized between the creation of this lambda + // and its execution, so we need to double check. + if (!this->Finalized) + if (auto Err = this->finalize()) + return std::move(Err); + return this->getSymbol(Name, false).getAddress(); + }; } void mapSectionAddress(const void *LocalAddress, @@ -169,9 +239,8 @@ private: } private: - - void buildInitialSymbolTable(const ObjectPtr &Obj) { - for (auto &Symbol : Obj->getBinary()->symbols()) { + void buildInitialSymbolTable(const OwnedObject &Obj) { + for (auto &Symbol : Obj.getBinary()->symbols()) { if (Symbol.getFlags() & object::SymbolRef::SF_Undefined) continue; Expected<StringRef> SymbolName = Symbol.getName(); @@ -186,65 +255,64 @@ private: } } - void updateSymbolTable(const RuntimeDyld &RTDyld) { - for (auto &SymEntry : SymbolTable) - SymEntry.second = RTDyld.getSymbol(SymEntry.first()); - } - // Contains the information needed prior to finalization: the object files, // memory manager, resolver, and flags needed for RuntimeDyld. struct PreFinalizeContents { - PreFinalizeContents(ObjectPtr Obj, SymbolResolverPtrT Resolver, - FinalizerFtor Finalizer, bool ProcessAllSections) - : Obj(std::move(Obj)), Resolver(std::move(Resolver)), - Finalizer(std::move(Finalizer)), - ProcessAllSections(ProcessAllSections) {} - - ObjectPtr Obj; - SymbolResolverPtrT Resolver; - FinalizerFtor Finalizer; + PreFinalizeContents(OwnedObject Obj, + std::shared_ptr<SymbolResolver> Resolver, + bool ProcessAllSections) + : Obj(std::move(Obj)), + Resolver(std::move(Resolver)), + ProcessAllSections(ProcessAllSections) {} + + OwnedObject Obj; + std::shared_ptr<SymbolResolver> Resolver; bool ProcessAllSections; - ObjHandleT Handle; - RuntimeDyld *RTDyld; + std::unique_ptr<RuntimeDyld> RTDyld; }; + VModuleKey K; + RTDyldObjectLinkingLayer &Parent; MemoryManagerPtrT MemMgr; + OwnedObject ObjForNotify; std::unique_ptr<PreFinalizeContents> PFC; }; - template <typename MemoryManagerPtrT, typename SymbolResolverPtrT, - typename FinalizerFtor> - std::unique_ptr< - ConcreteLinkedObject<MemoryManagerPtrT, SymbolResolverPtrT, FinalizerFtor>> - createLinkedObject(ObjectPtr Obj, MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver, - FinalizerFtor Finalizer, + template <typename MemoryManagerPtrT> + std::unique_ptr<ConcreteLinkedObject<MemoryManagerPtrT>> + createLinkedObject(RTDyldObjectLinkingLayer &Parent, VModuleKey K, + OwnedObject Obj, MemoryManagerPtrT MemMgr, + std::shared_ptr<SymbolResolver> Resolver, bool ProcessAllSections) { - using LOS = ConcreteLinkedObject<MemoryManagerPtrT, SymbolResolverPtrT, - FinalizerFtor>; - return llvm::make_unique<LOS>(std::move(Obj), std::move(MemMgr), - std::move(Resolver), std::move(Finalizer), + using LOS = ConcreteLinkedObject<MemoryManagerPtrT>; + return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj), + std::move(MemMgr), std::move(Resolver), ProcessAllSections); } public: + struct Resources { + std::shared_ptr<RuntimeDyld::MemoryManager> MemMgr; + std::shared_ptr<SymbolResolver> Resolver; + }; - /// @brief Functor for creating memory managers. - using MemoryManagerGetter = - std::function<std::shared_ptr<RuntimeDyld::MemoryManager>()>; + using ResourcesGetter = std::function<Resources(VModuleKey)>; - /// @brief Construct an ObjectLinkingLayer with the given NotifyLoaded, + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyFinalized functors. RTDyldObjectLinkingLayer( - MemoryManagerGetter GetMemMgr, + ExecutionSession &ES, ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(), - NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor()) - : GetMemMgr(GetMemMgr), + NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(), + NotifyFreedFtor NotifyFreed = NotifyFreedFtor()) + : ES(ES), GetResources(std::move(GetResources)), NotifyLoaded(std::move(NotifyLoaded)), NotifyFinalized(std::move(NotifyFinalized)), - ProcessAllSections(false) {} + NotifyFreed(std::move(NotifyFreed)), + ProcessAllSections(false) { + } - /// @brief Set the 'ProcessAllSections' flag. + /// Set the 'ProcessAllSections' flag. /// /// If set to true, all sections in each object file will be allocated using /// the memory manager, rather than just the sections required for execution. @@ -254,44 +322,26 @@ public: this->ProcessAllSections = ProcessAllSections; } - /// @brief Add an object to the JIT. - /// - /// @return A handle that can be used to refer to the loaded object (for - /// symbol searching, finalization, freeing memory, etc.). - Expected<ObjHandleT> addObject(ObjectPtr Obj, - std::shared_ptr<JITSymbolResolver> Resolver) { - auto Finalizer = [&](ObjHandleT H, RuntimeDyld &RTDyld, - const ObjectPtr &ObjToLoad, - std::function<void()> LOSHandleLoad) { - std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info = - RTDyld.loadObject(*ObjToLoad->getBinary()); - - LOSHandleLoad(); - - if (this->NotifyLoaded) - this->NotifyLoaded(H, ObjToLoad, *Info); + /// Add an object to the JIT. + Error addObject(VModuleKey K, ObjectPtr ObjBuffer) { - RTDyld.finalizeWithMemoryManagerLocking(); + auto Obj = + object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + if (!Obj) + return Obj.takeError(); - if (this->NotifyFinalized) - this->NotifyFinalized(H); - }; + assert(!LinkedObjects.count(K) && "VModuleKey already in use"); - auto LO = - createLinkedObject(std::move(Obj), GetMemMgr(), - std::move(Resolver), std::move(Finalizer), - ProcessAllSections); - // LOS is an owning-ptr. Keep a non-owning one so that we can set the handle - // below. - auto *LOPtr = LO.get(); + auto R = GetResources(K); - ObjHandleT Handle = LinkedObjList.insert(LinkedObjList.end(), std::move(LO)); - LOPtr->setHandle(Handle); + LinkedObjects[K] = createLinkedObject( + *this, K, OwnedObject(std::move(*Obj), std::move(ObjBuffer)), + std::move(R.MemMgr), std::move(R.Resolver), ProcessAllSections); - return Handle; + return Error::success(); } - /// @brief Remove the object associated with handle H. + /// Remove the object associated with VModuleKey K. /// /// All memory allocated for the object will be freed, and the sections and /// symbols it provided will no longer be available. No attempt is made to @@ -299,57 +349,64 @@ public: /// indirectly) will result in undefined behavior. If dependence tracking is /// required to detect or resolve such issues it should be added at a higher /// layer. - Error removeObject(ObjHandleT H) { + Error removeObject(VModuleKey K) { + assert(LinkedObjects.count(K) && "VModuleKey not associated with object"); // How do we invalidate the symbols in H? - LinkedObjList.erase(H); + LinkedObjects.erase(K); return Error::success(); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { - for (auto I = LinkedObjList.begin(), E = LinkedObjList.end(); I != E; - ++I) - if (auto Symbol = findSymbolIn(I, Name, ExportedSymbolsOnly)) - return Symbol; + for (auto &KV : LinkedObjects) + if (auto Sym = KV.second->getSymbol(Name, ExportedSymbolsOnly)) + return Sym; + else if (auto Err = Sym.takeError()) + return std::move(Err); return nullptr; } - /// @brief Search for the given named symbol in the context of the loaded - /// object represented by the handle H. - /// @param H The handle for the object to search in. + /// Search for the given named symbol in the context of the loaded + /// object represented by the VModuleKey K. + /// @param K The VModuleKey for the object to search in. /// @param Name The name of the symbol to search for. /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it is found in the /// given object. - JITSymbol findSymbolIn(ObjHandleT H, StringRef Name, + JITSymbol findSymbolIn(VModuleKey K, StringRef Name, bool ExportedSymbolsOnly) { - return (*H)->getSymbol(Name, ExportedSymbolsOnly); + assert(LinkedObjects.count(K) && "VModuleKey not associated with object"); + return LinkedObjects[K]->getSymbol(Name, ExportedSymbolsOnly); } - /// @brief Map section addresses for the object associated with the handle H. - void mapSectionAddress(ObjHandleT H, const void *LocalAddress, + /// Map section addresses for the object associated with the + /// VModuleKey K. + void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) { - (*H)->mapSectionAddress(LocalAddress, TargetAddr); + assert(LinkedObjects.count(K) && "VModuleKey not associated with object"); + LinkedObjects[K]->mapSectionAddress(LocalAddress, TargetAddr); } - /// @brief Immediately emit and finalize the object represented by the given - /// handle. - /// @param H Handle for object to emit/finalize. - Error emitAndFinalize(ObjHandleT H) { - (*H)->finalize(); - return Error::success(); + /// Immediately emit and finalize the object represented by the given + /// VModuleKey. + /// @param K VModuleKey for object to emit/finalize. + Error emitAndFinalize(VModuleKey K) { + assert(LinkedObjects.count(K) && "VModuleKey not associated with object"); + return LinkedObjects[K]->finalize(); } private: + ExecutionSession &ES; - LinkedObjectListT LinkedObjList; - MemoryManagerGetter GetMemMgr; + std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects; + ResourcesGetter GetResources; NotifyLoadedFtor NotifyLoaded; NotifyFinalizedFtor NotifyFinalized; + NotifyFreedFtor NotifyFreed; bool ProcessAllSections = false; }; diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h index 17255954a99f..955e77607a18 100644 --- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -306,8 +306,7 @@ public: using ObjHandleT = RemoteObjectLayerAPI::ObjHandleT; using RemoteSymbol = RemoteObjectLayerAPI::RemoteSymbol; - using ObjectPtr = - std::shared_ptr<object::OwningBinary<object::ObjectFile>>; + using ObjectPtr = std::unique_ptr<MemoryBuffer>; /// Create a RemoteObjectClientLayer that communicates with a /// RemoteObjectServerLayer instance via the given RPCEndpoint. @@ -323,15 +322,15 @@ public: *this, &ThisT::lookupInLogicalDylib); } - /// @brief Add an object to the JIT. + /// Add an object to the JIT. /// /// @return A handle that can be used to refer to the loaded object (for /// symbol searching, finalization, freeing memory, etc.). Expected<ObjHandleT> - addObject(ObjectPtr Object, std::shared_ptr<JITSymbolResolver> Resolver) { - StringRef ObjBuffer = Object->getBinary()->getData(); + addObject(ObjectPtr ObjBuffer, + std::shared_ptr<LegacyJITSymbolResolver> Resolver) { if (auto HandleOrErr = - this->Remote.template callB<AddObject>(ObjBuffer)) { + this->Remote.template callB<AddObject>(ObjBuffer->getBuffer())) { auto &Handle = *HandleOrErr; // FIXME: Return an error for this: assert(!Resolvers.count(Handle) && "Handle already in use?"); @@ -341,26 +340,26 @@ public: return HandleOrErr.takeError(); } - /// @brief Remove the given object from the JIT. + /// Remove the given object from the JIT. Error removeObject(ObjHandleT H) { return this->Remote.template callB<RemoveObject>(H); } - /// @brief Search for the given named symbol. + /// Search for the given named symbol. JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { return remoteToJITSymbol( this->Remote.template callB<FindSymbol>(Name, ExportedSymbolsOnly)); } - /// @brief Search for the given named symbol within the given context. + /// Search for the given named symbol within the given context. JITSymbol findSymbolIn(ObjHandleT H, StringRef Name, bool ExportedSymbolsOnly) { return remoteToJITSymbol( this->Remote.template callB<FindSymbolIn>(H, Name, ExportedSymbolsOnly)); } - /// @brief Immediately emit and finalize the object with the given handle. + /// Immediately emit and finalize the object with the given handle. Error emitAndFinalize(ObjHandleT H) { return this->Remote.template callB<EmitAndFinalize>(H); } @@ -386,7 +385,8 @@ private: } std::map<remote::ResourceIdMgr::ResourceId, - std::shared_ptr<JITSymbolResolver>> Resolvers; + std::shared_ptr<LegacyJITSymbolResolver>> + Resolvers; }; /// RemoteObjectServerLayer acts as a server and handling RPC calls for the @@ -459,30 +459,21 @@ private: Expected<ObjHandleT> addObject(std::string ObjBuffer) { auto Buffer = llvm::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); - if (auto ObjectOrErr = - object::ObjectFile::createObjectFile(Buffer->getMemBufferRef())) { - auto Object = - std::make_shared<object::OwningBinary<object::ObjectFile>>( - std::move(*ObjectOrErr), std::move(Buffer)); - - auto Id = HandleIdMgr.getNext(); - assert(!BaseLayerHandles.count(Id) && "Id already in use?"); - - auto Resolver = - createLambdaResolver( - [this, Id](const std::string &Name) { return lookup(Id, Name); }, - [this, Id](const std::string &Name) { - return lookupInLogicalDylib(Id, Name); - }); - - if (auto HandleOrErr = - BaseLayer.addObject(std::move(Object), std::move(Resolver))) { - BaseLayerHandles[Id] = std::move(*HandleOrErr); - return Id; - } else - return teeLog(HandleOrErr.takeError()); + auto Id = HandleIdMgr.getNext(); + assert(!BaseLayerHandles.count(Id) && "Id already in use?"); + + auto Resolver = createLambdaResolver( + [this, Id](const std::string &Name) { return lookup(Id, Name); }, + [this, Id](const std::string &Name) { + return lookupInLogicalDylib(Id, Name); + }); + + if (auto HandleOrErr = + BaseLayer.addObject(std::move(Buffer), std::move(Resolver))) { + BaseLayerHandles[Id] = std::move(*HandleOrErr); + return Id; } else - return teeLog(ObjectOrErr.takeError()); + return teeLog(HandleOrErr.takeError()); } Error removeObject(ObjHandleT H) { diff --git a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index b01fbd44bacd..4c45cfd199dd 100644 --- a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -23,29 +23,36 @@ namespace orc { class SymbolStringPtr; -/// @brief String pool for symbol names used by the JIT. +/// String pool for symbol names used by the JIT. class SymbolStringPool { friend class SymbolStringPtr; public: - /// @brief Create a symbol string pointer from the given string. + /// Destroy a SymbolStringPool. + ~SymbolStringPool(); + + /// Create a symbol string pointer from the given string. SymbolStringPtr intern(StringRef S); - /// @brief Remove from the pool any entries that are no longer referenced. + /// Remove from the pool any entries that are no longer referenced. void clearDeadEntries(); - /// @brief Returns true if the pool is empty. + /// Returns true if the pool is empty. bool empty() const; private: - using RefCountType = std::atomic<uint64_t>; + using RefCountType = std::atomic<size_t>; using PoolMap = StringMap<RefCountType>; using PoolMapEntry = StringMapEntry<RefCountType>; mutable std::mutex PoolMutex; PoolMap Pool; }; -/// @brief Pointer to a pooled string representing a symbol name. +/// Pointer to a pooled string representing a symbol name. class SymbolStringPtr { friend class SymbolStringPool; + friend bool operator==(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS); + friend bool operator<(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS); + public: SymbolStringPtr() = default; SymbolStringPtr(const SymbolStringPtr &Other) @@ -80,17 +87,7 @@ public: --S->getValue(); } - bool operator==(const SymbolStringPtr &Other) const { - return S == Other.S; - } - - bool operator!=(const SymbolStringPtr &Other) const { - return !(*this == Other); - } - - bool operator<(const SymbolStringPtr &Other) const { - return S->getValue() < Other.S->getValue(); - } + StringRef operator*() const { return S->first(); } private: @@ -103,25 +100,39 @@ private: SymbolStringPool::PoolMapEntry *S = nullptr; }; +inline bool operator==(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { + return LHS.S == RHS.S; +} + +inline bool operator!=(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { + return !(LHS == RHS); +} + +inline bool operator<(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { + return LHS.S < RHS.S; +} + +inline SymbolStringPool::~SymbolStringPool() { +#ifndef NDEBUG + clearDeadEntries(); + assert(Pool.empty() && "Dangling references at pool destruction time"); +#endif // NDEBUG +} + inline SymbolStringPtr SymbolStringPool::intern(StringRef S) { std::lock_guard<std::mutex> Lock(PoolMutex); - auto I = Pool.find(S); - if (I != Pool.end()) - return SymbolStringPtr(&*I); - + PoolMap::iterator I; bool Added; std::tie(I, Added) = Pool.try_emplace(S, 0); - assert(Added && "Insert should always succeed here"); return SymbolStringPtr(&*I); } inline void SymbolStringPool::clearDeadEntries() { std::lock_guard<std::mutex> Lock(PoolMutex); for (auto I = Pool.begin(), E = Pool.end(); I != E;) { - auto Tmp = std::next(I); - if (I->second == 0) - Pool.erase(I); - I = Tmp; + auto Tmp = I++; + if (Tmp->second == 0) + Pool.erase(Tmp); } } |