diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h')
| -rw-r--r-- | llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h | 534 |
1 files changed, 534 insertions, 0 deletions
diff --git a/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h new file mode 100644 index 000000000000..e0af3df9d010 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -0,0 +1,534 @@ +//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H +#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H + +#include "llvm-c/OrcBindings.h" +#include "llvm-c/TargetMachine.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include <algorithm> +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace llvm { + +class OrcCBindingsStack; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +namespace detail { + +// FIXME: Kill this off once the Layer concept becomes an interface. +class GenericLayer { +public: + virtual ~GenericLayer() = default; + + virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) = 0; + virtual Error removeModule(orc::VModuleKey K) = 0; + }; + + template <typename LayerT> class GenericLayerImpl : public GenericLayer { + public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) override { + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); + } + + Error removeModule(orc::VModuleKey K) override { + return Layer.removeModule(K); + } + + private: + LayerT &Layer; + }; + + template <> + class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer { + private: + using LayerT = orc::LegacyRTDyldObjectLinkingLayer; + public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) override { + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); + } + + Error removeModule(orc::VModuleKey K) override { + return Layer.removeObject(K); + } + + private: + LayerT &Layer; + }; + + template <typename LayerT> + std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { + return std::make_unique<GenericLayerImpl<LayerT>>(Layer); + } + +} // end namespace detail + +class OrcCBindingsStack { +public: + + using CompileCallbackMgr = orc::JITCompileCallbackManager; + using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer; + using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>; + using CODLayerT = + orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>; + + using CallbackManagerBuilder = + std::function<std::unique_ptr<CompileCallbackMgr>()>; + + using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT; + +private: + + using OwningObject = object::OwningBinary<object::ObjectFile>; + + class CBindingsResolver : public orc::SymbolResolver { + public: + CBindingsResolver(OrcCBindingsStack &Stack, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) + : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), + ExternalResolverCtx(std::move(ExternalResolverCtx)) {} + + orc::SymbolNameSet + getResponsibilitySet(const orc::SymbolNameSet &Symbols) override { + orc::SymbolNameSet Result; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (!Sym.getFlags().isStrong()) + Result.insert(S); + } else if (auto Err = Sym.takeError()) { + Stack.reportError(std::move(Err)); + return orc::SymbolNameSet(); + } + } + + return Result; + } + + orc::SymbolNameSet + lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query, + orc::SymbolNameSet Symbols) override { + orc::SymbolNameSet UnresolvedSymbols; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + } else { + Stack.ES.legacyFailQuery(*Query, Addr.takeError()); + return orc::SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + Stack.ES.legacyFailQuery(*Query, std::move(Err)); + return orc::SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + + if (Query->isComplete()) + Query->handleComplete(); + + return UnresolvedSymbols; + } + + private: + JITSymbol findSymbol(const std::string &Name) { + // Search order: + // 1. JIT'd symbols. + // 2. Runtime overrides. + // 3. External resolver (if present). + + if (Stack.CODLayer) { + if (auto Sym = Stack.CODLayer->findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } else { + if (auto Sym = Stack.CompileLayer.findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } + + if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) + return Sym; + + if (ExternalResolver) + return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx), + JITSymbolFlags::Exported); + + return JITSymbol(nullptr); + } + + OrcCBindingsStack &Stack; + LLVMOrcSymbolResolverFn ExternalResolver; + void *ExternalResolverCtx = nullptr; + }; + +public: + OrcCBindingsStack(TargetMachine &TM, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder) + : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), + IndirectStubsMgr(IndirectStubsMgrBuilder()), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + orc::SimpleCompiler(TM)), + CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), + std::move(IndirectStubsMgrBuilder), Resolvers)), + CXXRuntimeOverrides( + AcknowledgeORCv1Deprecation, + [this](const std::string &S) { return mangle(S); }) {} + + Error shutdown() { + // Run any destructors registered with __cxa_atexit. + CXXRuntimeOverrides.runDestructors(); + // Run any IR destructors. + for (auto &DtorRunner : IRStaticDestructorRunners) + if (auto Err = DtorRunner.runViaLayer(*this)) + return Err; + return Error::success(); + } + + std::string mangle(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return MangledName; + } + + template <typename PtrTy> + static PtrTy fromTargetAddress(JITTargetAddress Addr) { + return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr)); + } + + Expected<JITTargetAddress> + createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, + void *CallbackCtx) { + auto WrappedCallback = [=]() -> JITTargetAddress { + return Callback(wrap(this), CallbackCtx); + }; + + return CCMgr->getCompileCallback(std::move(WrappedCallback)); + } + + Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) { + return IndirectStubsMgr->createStub(StubName, Addr, + JITSymbolFlags::Exported); + } + + Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) { + return IndirectStubsMgr->updatePointer(Name, Addr); + } + + template <typename LayerT> + Expected<orc::VModuleKey> + addIRModule(LayerT &Layer, std::unique_ptr<Module> M, + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + + // Attach a data-layout if one isn't already present. + if (M->getDataLayout().isDefault()) + M->setDataLayout(DL); + + // Record the static constructors and destructors. We have to do this before + // we hand over ownership of the module to the JIT. + std::vector<std::string> CtorNames, DtorNames; + for (auto Ctor : orc::getConstructors(*M)) + CtorNames.push_back(mangle(Ctor.Func->getName())); + for (auto Dtor : orc::getDestructors(*M)) + DtorNames.push_back(mangle(Dtor.Func->getName())); + + // Add the module to the JIT. + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver, + ExternalResolverCtx); + if (auto Err = Layer.addModule(K, std::move(M))) + return std::move(Err); + + KeyLayers[K] = detail::createGenericLayer(Layer); + + // Run the static constructors, and save the static destructor runner for + // execution when the JIT is torn down. + orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner( + AcknowledgeORCv1Deprecation, std::move(CtorNames), K); + if (auto Err = CtorRunner.runViaLayer(*this)) + return std::move(Err); + + IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation, + std::move(DtorNames), K); + + return K; + } + + Expected<orc::VModuleKey> + addIRModuleEager(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(CompileLayer, std::move(M), + std::make_unique<SectionMemoryManager>(), + std::move(ExternalResolver), ExternalResolverCtx); + } + + Expected<orc::VModuleKey> + addIRModuleLazy(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + if (!CODLayer) + return make_error<StringError>("Can not add lazy module: No compile " + "callback manager available", + inconvertibleErrorCode()); + + return addIRModule(*CODLayer, std::move(M), + std::make_unique<SectionMemoryManager>(), + std::move(ExternalResolver), ExternalResolverCtx); + } + + Error removeModule(orc::VModuleKey K) { + // FIXME: Should error release the module key? + if (auto Err = KeyLayers[K]->removeModule(K)) + return Err; + ES.releaseVModule(K); + KeyLayers.erase(K); + return Error::success(); + } + + Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + if (auto Obj = object::ObjectFile::createObjectFile( + ObjBuffer->getMemBufferRef())) { + + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); + + if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer))) + return std::move(Err); + + KeyLayers[K] = detail::createGenericLayer(ObjectLayer); + + return K; + } else + return Obj.takeError(); + } + + JITSymbol findSymbol(const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) + return Sym; + if (CODLayer) + return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly); + return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); + } + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) { + assert(KeyLayers.count(K) && "looking up symbol in unknown module"); + return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); + } + + Expected<JITTargetAddress> findSymbolAddress(const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return std::move(Err); + } + + // No symbol not found. Return 0. + return 0; + } + + Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K, + const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return std::move(Err); + } + + // Symbol not found. Return 0. + return 0; + } + + const std::string &getErrorMessage() const { return ErrMsg; } + + void RegisterJITEventListener(JITEventListener *L) { + if (!L) + return; + EventListeners.push_back(L); + } + + void UnregisterJITEventListener(JITEventListener *L) { + if (!L) + return; + + auto I = find(reverse(EventListeners), L); + if (I != EventListeners.rend()) { + std::swap(*I, EventListeners.back()); + EventListeners.pop_back(); + } + } + +private: + using ResolverMap = + std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>; + + static std::unique_ptr<CompileCallbackMgr> + createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) { + auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0); + if (!CCMgr) { + // FIXME: It would be good if we could report this somewhere, but we do + // have an instance yet. + logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: "); + return nullptr; + } + return std::move(*CCMgr); + } + + static std::unique_ptr<CODLayerT> + createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer, + CompileCallbackMgr *CCMgr, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder, + ResolverMap &Resolvers) { + // If there is no compile callback manager available we can not create a + // compile on demand layer. + if (!CCMgr) + return nullptr; + + return std::make_unique<CODLayerT>( + AcknowledgeORCv1Deprecation, ES, CompileLayer, + [&Resolvers](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && "No resolver for module K"); + return ResolverI->second; + }, + [&Resolvers](orc::VModuleKey K, + std::shared_ptr<orc::SymbolResolver> Resolver) { + assert(!Resolvers.count(K) && "Resolver already present"); + Resolvers[K] = std::move(Resolver); + }, + [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr, + std::move(IndirectStubsMgrBuilder), false); + } + + void reportError(Error Err) { + // FIXME: Report errors on the execution session. + logAllUnhandledErrors(std::move(Err), errs(), "ORC error: "); + }; + + void notifyFinalized(orc::VModuleKey K, + const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); + for (auto &Listener : EventListeners) + Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo); + } + + void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); + for (auto &Listener : EventListeners) + Listener->notifyFreeingObject(Key); + } + + orc::ExecutionSession ES; + std::unique_ptr<CompileCallbackMgr> CCMgr; + + std::vector<JITEventListener *> EventListeners; + + DataLayout DL; + SectionMemoryManager CCMgrMemMgr; + + std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; + + ObjLayerT ObjectLayer; + CompileLayerT CompileLayer; + std::unique_ptr<CODLayerT> CODLayer; + + std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; + + orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides; + std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; + std::string ErrMsg; + + ResolverMap Resolvers; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H |