diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2018-07-28 10:51:19 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2018-07-28 10:51:19 +0000 |
commit | eb11fae6d08f479c0799db45860a98af528fa6e7 (patch) | |
tree | 44d492a50c8c1a7eb8e2d17ea3360ec4d066f042 /lib/ExecutionEngine/Orc | |
parent | b8a2042aa938069e862750553db0e4d82d25822c (diff) |
Notes
Diffstat (limited to 'lib/ExecutionEngine/Orc')
19 files changed, 3364 insertions, 250 deletions
diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt index f83e002c758f..a7500ef20f37 100644 --- a/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -1,12 +1,21 @@ add_llvm_library(LLVMOrcJIT + CompileOnDemandLayer.cpp + Core.cpp ExecutionUtils.cpp IndirectionUtils.cpp + IRCompileLayer.cpp + IRTransformLayer.cpp + Legacy.cpp + Layer.cpp + LLJIT.cpp NullResolver.cpp + ObjectTransformLayer.cpp OrcABISupport.cpp OrcCBindings.cpp OrcError.cpp OrcMCJITReplacement.cpp RPCUtils.cpp + RTDyldObjectLinkingLayer.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc @@ -14,3 +23,9 @@ add_llvm_library(LLVMOrcJIT DEPENDS intrinsics_gen ) + +target_link_libraries(LLVMOrcJIT + PRIVATE + LLVMBitReader + LLVMBitWriter + ) diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp new file mode 100644 index 000000000000..d42e7b05ba67 --- /dev/null +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -0,0 +1,343 @@ +//===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/Cloning.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +template <typename MaterializerFtor> +class LambdaValueMaterializer final : public ValueMaterializer { +public: + LambdaValueMaterializer(MaterializerFtor M) : M(std::move(M)) {} + + Value *materialize(Value *V) final { return M(V); } + +private: + MaterializerFtor M; +}; + +template <typename MaterializerFtor> +LambdaValueMaterializer<MaterializerFtor> +createLambdaValueMaterializer(MaterializerFtor M) { + return LambdaValueMaterializer<MaterializerFtor>(std::move(M)); +} +} // namespace + +static void extractAliases(MaterializationResponsibility &R, Module &M, + MangleAndInterner &Mangle) { + SymbolAliasMap Aliases; + + std::vector<GlobalAlias *> ModAliases; + for (auto &A : M.aliases()) + ModAliases.push_back(&A); + + for (auto *A : ModAliases) { + Constant *Aliasee = A->getAliasee(); + assert(A->hasName() && "Anonymous alias?"); + assert(Aliasee->hasName() && "Anonymous aliasee"); + std::string AliasName = A->getName(); + + Aliases[Mangle(AliasName)] = SymbolAliasMapEntry( + {Mangle(Aliasee->getName()), JITSymbolFlags::fromGlobalValue(*A)}); + + if (isa<Function>(Aliasee)) { + auto *F = cloneFunctionDecl(M, *cast<Function>(Aliasee)); + A->replaceAllUsesWith(F); + A->eraseFromParent(); + F->setName(AliasName); + } else if (isa<GlobalValue>(Aliasee)) { + auto *G = cloneGlobalVariableDecl(M, *cast<GlobalVariable>(Aliasee)); + A->replaceAllUsesWith(G); + A->eraseFromParent(); + G->setName(AliasName); + } + } + + R.replace(symbolAliases(std::move(Aliases))); +} + +static std::unique_ptr<Module> +extractAndClone(Module &M, LLVMContext &NewContext, StringRef Suffix, + function_ref<bool(const GlobalValue *)> ShouldCloneDefinition) { + SmallVector<char, 1> ClonedModuleBuffer; + + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDefinition(GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); + + for (auto *GV : ClonedDefsInSrc) { + // Delete the definition and bump the linkage in the source module. + if (isa<Function>(GV)) { + auto &F = *cast<Function>(GV); + F.deleteBody(); + F.setPersonalityFn(nullptr); + } else if (isa<GlobalVariable>(GV)) { + cast<GlobalVariable>(GV)->setInitializer(nullptr); + } else + llvm_unreachable("Unsupported global type"); + + GV->setLinkage(GlobalValue::ExternalLinkage); + } + + BitcodeWriter BCWriter(ClonedModuleBuffer); + + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } + + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + + auto ClonedModule = + cantFail(parseBitcodeFile(ClonedModuleBufferRef, NewContext)); + ClonedModule->setModuleIdentifier((M.getName() + Suffix).str()); + return ClonedModule; +} + +static std::unique_ptr<Module> extractGlobals(Module &M, + LLVMContext &NewContext) { + return extractAndClone(M, NewContext, ".globals", [](const GlobalValue *GV) { + return isa<GlobalVariable>(GV); + }); +} + +namespace llvm { +namespace orc { + +class ExtractingIRMaterializationUnit : public IRMaterializationUnit { +public: + ExtractingIRMaterializationUnit(ExecutionSession &ES, + CompileOnDemandLayer2 &Parent, + std::unique_ptr<Module> M) + : IRMaterializationUnit(ES, std::move(M)), Parent(Parent) {} + + ExtractingIRMaterializationUnit(std::unique_ptr<Module> M, + SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition, + CompileOnDemandLayer2 &Parent) + : IRMaterializationUnit(std::move(M), std::move(SymbolFlags), + std::move(SymbolToDefinition)), + Parent(Parent) {} + +private: + void materialize(MaterializationResponsibility R) override { + // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the + // extracted module key, extracted module, and source module key + // together. This could be used, for example, to provide a specific + // memory manager instance to the linking layer. + + auto RequestedSymbols = R.getRequestedSymbols(); + + // Extract the requested functions into a new module. + std::unique_ptr<Module> ExtractedFunctionsModule; + if (!RequestedSymbols.empty()) { + std::string Suffix; + std::set<const GlobalValue *> FunctionsToClone; + for (auto &Name : RequestedSymbols) { + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && I->second != nullptr && + "Should have a non-null definition"); + FunctionsToClone.insert(I->second); + Suffix += "."; + Suffix += *Name; + } + + std::lock_guard<std::mutex> Lock(SourceModuleMutex); + ExtractedFunctionsModule = + extractAndClone(*M, Parent.GetAvailableContext(), Suffix, + [&](const GlobalValue *GV) -> bool { + return FunctionsToClone.count(GV); + }); + } + + // Build a new ExtractingIRMaterializationUnit to delegate the unrequested + // symbols to. + SymbolFlagsMap DelegatedSymbolFlags; + IRMaterializationUnit::SymbolNameToDefinitionMap + DelegatedSymbolToDefinition; + for (auto &KV : SymbolToDefinition) { + if (RequestedSymbols.count(KV.first)) + continue; + DelegatedSymbolFlags[KV.first] = + JITSymbolFlags::fromGlobalValue(*KV.second); + DelegatedSymbolToDefinition[KV.first] = KV.second; + } + + if (!DelegatedSymbolFlags.empty()) { + assert(DelegatedSymbolFlags.size() == + DelegatedSymbolToDefinition.size() && + "SymbolFlags and SymbolToDefinition should have the same number " + "of entries"); + R.replace(llvm::make_unique<ExtractingIRMaterializationUnit>( + std::move(M), std::move(DelegatedSymbolFlags), + std::move(DelegatedSymbolToDefinition), Parent)); + } + + if (ExtractedFunctionsModule) + Parent.emitExtractedFunctionsModule(std::move(R), + std::move(ExtractedFunctionsModule)); + } + + void discard(const VSO &V, SymbolStringPtr Name) override { + // All original symbols were materialized by the CODLayer and should be + // final. The function bodies provided by M should never be overridden. + llvm_unreachable("Discard should never be called on an " + "ExtractingIRMaterializationUnit"); + } + + mutable std::mutex SourceModuleMutex; + CompileOnDemandLayer2 &Parent; +}; + +CompileOnDemandLayer2::CompileOnDemandLayer2( + ExecutionSession &ES, IRLayer &BaseLayer, JITCompileCallbackManager &CCMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager, + GetAvailableContextFunction GetAvailableContext) + : IRLayer(ES), BaseLayer(BaseLayer), CCMgr(CCMgr), + BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)), + GetAvailableContext(std::move(GetAvailableContext)) {} + +Error CompileOnDemandLayer2::add(VSO &V, VModuleKey K, + std::unique_ptr<Module> M) { + return IRLayer::add(V, K, std::move(M)); +} + +void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + auto &ES = getExecutionSession(); + assert(M && "M should not be null"); + + for (auto &GV : M->global_values()) + if (GV.hasWeakLinkage()) + GV.setLinkage(GlobalValue::ExternalLinkage); + + MangleAndInterner Mangle(ES, M->getDataLayout()); + + extractAliases(R, *M, Mangle); + + auto GlobalsModule = extractGlobals(*M, GetAvailableContext()); + + // Delete the bodies of any available externally functions, rename the + // rest, and build the compile callbacks. + std::map<SymbolStringPtr, std::pair<JITTargetAddress, JITSymbolFlags>> + StubCallbacksAndLinkages; + auto &TargetVSO = R.getTargetVSO(); + + for (auto &F : M->functions()) { + if (F.isDeclaration()) + continue; + + if (F.hasAvailableExternallyLinkage()) { + F.deleteBody(); + F.setPersonalityFn(nullptr); + continue; + } + + assert(F.hasName() && "Function should have a name"); + std::string StubUnmangledName = F.getName(); + F.setName(F.getName() + "$body"); + auto StubDecl = cloneFunctionDecl(*M, F); + StubDecl->setName(StubUnmangledName); + StubDecl->setPersonalityFn(nullptr); + StubDecl->setLinkage(GlobalValue::ExternalLinkage); + F.replaceAllUsesWith(StubDecl); + + auto StubName = Mangle(StubUnmangledName); + auto BodyName = Mangle(F.getName()); + if (auto CallbackAddr = CCMgr.getCompileCallback( + [BodyName, &TargetVSO, &ES]() -> JITTargetAddress { + if (auto Sym = lookup({&TargetVSO}, BodyName)) + return Sym->getAddress(); + else { + ES.reportError(Sym.takeError()); + return 0; + } + })) { + auto Flags = JITSymbolFlags::fromGlobalValue(F); + Flags &= ~JITSymbolFlags::Weak; + StubCallbacksAndLinkages[std::move(StubName)] = + std::make_pair(*CallbackAddr, Flags); + } else { + ES.reportError(CallbackAddr.takeError()); + R.failMaterialization(); + return; + } + } + + // Build the stub inits map. + IndirectStubsManager::StubInitsMap StubInits; + for (auto &KV : StubCallbacksAndLinkages) + StubInits[*KV.first] = KV.second; + + // Build the function-body-extracting materialization unit. + if (auto Err = R.getTargetVSO().define( + llvm::make_unique<ExtractingIRMaterializationUnit>(ES, *this, + std::move(M)))) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + // Build the stubs. + // FIXME: Remove function bodies materialization unit if stub creation fails. + auto &StubsMgr = getStubsManager(TargetVSO); + if (auto Err = StubsMgr.createStubs(StubInits)) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + // Resolve and finalize stubs. + SymbolMap ResolvedStubs; + for (auto &KV : StubCallbacksAndLinkages) { + if (auto Sym = StubsMgr.findStub(*KV.first, false)) + ResolvedStubs[KV.first] = Sym; + else + llvm_unreachable("Stub went missing"); + } + + R.resolve(ResolvedStubs); + + BaseLayer.emit(std::move(R), std::move(K), std::move(GlobalsModule)); +} + +IndirectStubsManager &CompileOnDemandLayer2::getStubsManager(const VSO &V) { + std::lock_guard<std::mutex> Lock(CODLayerMutex); + StubManagersMap::iterator I = StubsMgrs.find(&V); + if (I == StubsMgrs.end()) + I = StubsMgrs.insert(std::make_pair(&V, BuildIndirectStubsManager())).first; + return *I->second; +} + +void CompileOnDemandLayer2::emitExtractedFunctionsModule( + MaterializationResponsibility R, std::unique_ptr<Module> M) { + auto K = getExecutionSession().allocateVModule(); + BaseLayer.emit(std::move(R), std::move(K), std::move(M)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp new file mode 100644 index 000000000000..4325d57f73d0 --- /dev/null +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -0,0 +1,1690 @@ +//===----- Core.cpp - Core ORC APIs (MaterializationUnit, VSO, etc.) ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" + +#if LLVM_ENABLE_THREADS +#include <future> +#endif + +namespace llvm { +namespace orc { + +char FailedToMaterialize::ID = 0; +char SymbolsNotFound::ID = 0; + +RegisterDependenciesFunction NoDependenciesToRegister = + RegisterDependenciesFunction(); + +void MaterializationUnit::anchor() {} + +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.isWeak()) + OS << 'W'; + else if (Flags.isCommon()) + OS << 'C'; + else + OS << 'S'; + + if (Flags.isExported()) + OS << 'E'; + else + OS << 'H'; + + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { + OS << format("0x%016x", Sym.getAddress()) << " " << Sym.getFlags(); + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { + OS << "\"" << *KV.first << "\": " << KV.second; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + OS << "{"; + if (!Symbols.empty()) { + OS << " \"" << **Symbols.begin() << "\""; + for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) + OS << ", \"" << *Sym << "\""; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { + OS << "{"; + if (!Symbols.empty()) { + OS << " {" << *Symbols.begin() << "}"; + for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) + OS << ", {" << Sym << "}"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + OS << "{"; + if (!SymbolFlags.empty()) { + OS << " {\"" << *SymbolFlags.begin()->first + << "\": " << SymbolFlags.begin()->second << "}"; + for (auto &KV : + make_range(std::next(SymbolFlags.begin()), SymbolFlags.end())) + OS << ", {\"" << *KV.first << "\": " << KV.second << "}"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { + OS << "{"; + if (!Deps.empty()) { + OS << " { " << Deps.begin()->first->getName() << ": " + << Deps.begin()->second << " }"; + for (auto &KV : make_range(std::next(Deps.begin()), Deps.end())) + OS << ", { " << KV.first->getName() << ": " << KV.second << " }"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const VSOList &VSOs) { + OS << "["; + if (!VSOs.empty()) { + assert(VSOs.front() && "VSOList entries must not be null"); + OS << " " << VSOs.front()->getName(); + for (auto *V : make_range(std::next(VSOs.begin()), VSOs.end())) { + assert(V && "VSOList entries must not be null"); + OS << ", " << V->getName(); + } + } + OS << " ]"; + return OS; +} + +FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code FailedToMaterialize::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void FailedToMaterialize::log(raw_ostream &OS) const { + OS << "Failed to materialize symbols: " << Symbols; +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsNotFound::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsNotFound::log(raw_ostream &OS) const { + OS << "Symbols not found: " << Symbols; +} + +void ExecutionSessionBase::legacyFailQuery(AsynchronousSymbolQuery &Q, + Error Err) { + assert(!!Err && "Error should be in failure state"); + + bool SendErrorToQuery; + runSessionLocked([&]() { + Q.detach(); + SendErrorToQuery = Q.canStillFail(); + }); + + if (SendErrorToQuery) + Q.handleFailed(std::move(Err)); + else + reportError(std::move(Err)); +} + +Expected<SymbolMap> ExecutionSessionBase::legacyLookup( + ExecutionSessionBase &ES, LegacyAsyncLookupFunction AsyncLookup, + SymbolNameSet Names, bool WaitUntilReady, + RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + ES.reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + ES.reportError(std::move(Err)); + }; + } +#endif + + auto Query = std::make_shared<AsynchronousSymbolQuery>( + Names, std::move(OnResolve), std::move(OnReady)); + // FIXME: This should be run session locked along with the registration code + // and error reporting below. + SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); + + // If the query was lodged successfully then register the dependencies, + // otherwise fail it with an error. + if (UnresolvedSymbols.empty()) + RegisterDependencies(Query->QueryRegistrations); + else { + bool DeliverError = runSessionLocked([&]() { + Query->detach(); + return Query->canStillFail(); + }); + auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); + if (DeliverError) + Query->handleFailed(std::move(Err)); + else + ES.reportError(std::move(Err)); + } + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif +} + +void ExecutionSessionBase::lookup( + const VSOList &VSOs, const SymbolNameSet &Symbols, + SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + RegisterDependenciesFunction RegisterDependencies) { + + // lookup can be re-entered recursively if running on a single thread. Run any + // outstanding MUs in case this query depends on them, otherwise the main + // thread will starve waiting for a result from an MU that it failed to run. + runOutstandingMUs(); + + auto Unresolved = std::move(Symbols); + std::map<VSO *, MaterializationUnitList> MUsMap; + auto Q = std::make_shared<AsynchronousSymbolQuery>( + Symbols, std::move(OnResolve), std::move(OnReady)); + bool QueryIsFullyResolved = false; + bool QueryIsFullyReady = false; + bool QueryFailed = false; + + runSessionLocked([&]() { + for (auto *V : VSOs) { + assert(V && "VSOList entries must not be null"); + assert(!MUsMap.count(V) && + "VSOList should not contain duplicate entries"); + V->lodgeQuery(Q, Unresolved, MUsMap[V]); + } + + if (Unresolved.empty()) { + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryIsFullyResolved = Q->isFullyResolved(); + QueryIsFullyReady = Q->isFullyReady(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + } else { + // Query failed due to unresolved symbols. + QueryFailed = true; + + // Disconnect the query from its dependencies. + Q->detach(); + + // Replace the MUs. + for (auto &KV : MUsMap) + for (auto &MU : KV.second) + KV.first->replace(std::move(MU)); + } + }); + + if (QueryFailed) { + Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); + return; + } else { + if (QueryIsFullyResolved) + Q->handleFullyResolved(); + if (QueryIsFullyReady) + Q->handleFullyReady(); + } + + // Move the MUs to the OutstandingMUs list, then materialize. + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + for (auto &KV : MUsMap) + for (auto &MU : KV.second) + OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); + } + + runOutstandingMUs(); +} + +Expected<SymbolMap> +ExecutionSessionBase::lookup(const VSOList &VSOs, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, + bool WaitUntilReady) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } +#endif + + // Perform the asynchronous lookup. + lookup(VSOs, Symbols, OnResolve, OnReady, RegisterDependencies); + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif +} + +void ExecutionSessionBase::runOutstandingMUs() { + while (1) { + std::pair<VSO *, std::unique_ptr<MaterializationUnit>> VSOAndMU; + + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + if (!OutstandingMUs.empty()) { + VSOAndMU = std::move(OutstandingMUs.back()); + OutstandingMUs.pop_back(); + } + } + + if (VSOAndMU.first) { + assert(VSOAndMU.second && "VSO, but no MU?"); + dispatchMaterialization(*VSOAndMU.first, std::move(VSOAndMU.second)); + } else + break; + } +} + +AsynchronousSymbolQuery::AsynchronousSymbolQuery( + const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved, + SymbolsReadyCallback NotifySymbolsReady) + : NotifySymbolsResolved(std::move(NotifySymbolsResolved)), + NotifySymbolsReady(std::move(NotifySymbolsReady)) { + NotYetResolvedCount = NotYetReadyCount = Symbols.size(); + + for (auto &S : Symbols) + ResolvedSymbols[S] = nullptr; +} + +void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, + JITEvaluatedSymbol Sym) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Resolving symbol outside the requested set"); + assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + I->second = std::move(Sym); + --NotYetResolvedCount; +} + +void AsynchronousSymbolQuery::handleFullyResolved() { + assert(NotYetResolvedCount == 0 && "Not fully resolved?"); + assert(NotifySymbolsResolved && + "NotifySymbolsResolved already called or error occurred"); + NotifySymbolsResolved(std::move(ResolvedSymbols)); + NotifySymbolsResolved = SymbolsResolvedCallback(); +} + +void AsynchronousSymbolQuery::notifySymbolReady() { + assert(NotYetReadyCount != 0 && "All symbols already finalized"); + --NotYetReadyCount; +} + +void AsynchronousSymbolQuery::handleFullyReady() { + assert(QueryRegistrations.empty() && + "Query is still registered with some symbols"); + assert(!NotifySymbolsResolved && "Resolution not applied yet"); + NotifySymbolsReady(Error::success()); + NotifySymbolsReady = SymbolsReadyCallback(); +} + +bool AsynchronousSymbolQuery::canStillFail() { + return (NotifySymbolsResolved || NotifySymbolsReady); +} + +void AsynchronousSymbolQuery::handleFailed(Error Err) { + assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && + NotYetResolvedCount == 0 && NotYetReadyCount == 0 && + "Query should already have been abandoned"); + if (NotifySymbolsResolved) { + NotifySymbolsResolved(std::move(Err)); + NotifySymbolsResolved = SymbolsResolvedCallback(); + } else { + assert(NotifySymbolsReady && "Failed after both callbacks issued?"); + NotifySymbolsReady(std::move(Err)); + } + NotifySymbolsReady = SymbolsReadyCallback(); +} + +void AsynchronousSymbolQuery::addQueryDependence(VSO &V, SymbolStringPtr Name) { + bool Added = QueryRegistrations[&V].insert(std::move(Name)).second; + (void)Added; + assert(Added && "Duplicate dependence notification?"); +} + +void AsynchronousSymbolQuery::removeQueryDependence( + VSO &V, const SymbolStringPtr &Name) { + auto QRI = QueryRegistrations.find(&V); + assert(QRI != QueryRegistrations.end() && "No dependencies registered for V"); + assert(QRI->second.count(Name) && "No dependency on Name in V"); + QRI->second.erase(Name); + if (QRI->second.empty()) + QueryRegistrations.erase(QRI); +} + +void AsynchronousSymbolQuery::detach() { + ResolvedSymbols.clear(); + NotYetResolvedCount = 0; + NotYetReadyCount = 0; + for (auto &KV : QueryRegistrations) + KV.first->detachQueryHelper(*this, KV.second); + QueryRegistrations.clear(); +} + +MaterializationResponsibility::MaterializationResponsibility( + VSO &V, SymbolFlagsMap SymbolFlags) + : V(V), SymbolFlags(std::move(SymbolFlags)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + +#ifndef NDEBUG + for (auto &KV : this->SymbolFlags) + KV.second |= JITSymbolFlags::Materializing; +#endif +} + +MaterializationResponsibility::~MaterializationResponsibility() { + assert(SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); +} + +SymbolNameSet MaterializationResponsibility::getRequestedSymbols() { + return V.getRequestedSymbols(SymbolFlags); +} + +void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto I = SymbolFlags.find(KV.first); + assert(I != SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + assert(I->second.isMaterializing() && "Duplicate resolution"); + I->second &= ~JITSymbolFlags::Materializing; + if (I->second.isWeak()) + assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && + "Resolving symbol with incorrect flags"); + else + assert(I->second == KV.second.getFlags() && + "Resolving symbol with incorrect flags"); + } +#endif + + V.resolve(Symbols); +} + +void MaterializationResponsibility::finalize() { +#ifndef NDEBUG + for (auto &KV : SymbolFlags) + assert(!KV.second.isMaterializing() && + "Failed to resolve symbol before finalization"); +#endif // NDEBUG + + V.finalize(SymbolFlags); + SymbolFlags.clear(); +} + +Error MaterializationResponsibility::defineMaterializing( + const SymbolFlagsMap &NewSymbolFlags) { + // Add the given symbols to this responsibility object. + // It's ok if we hit a duplicate here: In that case the new version will be + // discarded, and the VSO::defineMaterializing method will return a duplicate + // symbol error. + for (auto &KV : NewSymbolFlags) { + auto I = SymbolFlags.insert(KV).first; + (void)I; +#ifndef NDEBUG + I->second |= JITSymbolFlags::Materializing; +#endif + } + + return V.defineMaterializing(NewSymbolFlags); +} + +void MaterializationResponsibility::failMaterialization() { + + SymbolNameSet FailedSymbols; + for (auto &KV : SymbolFlags) + FailedSymbols.insert(KV.first); + + V.notifyFailed(FailedSymbols); + SymbolFlags.clear(); +} + +void MaterializationResponsibility::replace( + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) + SymbolFlags.erase(KV.first); + + V.replace(std::move(MU)); +} + +MaterializationResponsibility +MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) { + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = SymbolFlags.find(Name); + assert(I != SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + SymbolFlags.erase(I); + } + + return MaterializationResponsibility(V, std::move(DelegatedFlags)); +} + +void MaterializationResponsibility::addDependencies( + const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { + assert(SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + V.addDependencies(Name, Dependencies); +} + +void MaterializationResponsibility::addDependenciesForAll( + const SymbolDependenceMap &Dependencies) { + for (auto &KV : SymbolFlags) + V.addDependencies(KV.first, Dependencies); +} + +AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( + SymbolMap Symbols) + : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {} + +void AbsoluteSymbolsMaterializationUnit::materialize( + MaterializationResponsibility R) { + R.resolve(Symbols); + R.finalize(); +} + +void AbsoluteSymbolsMaterializationUnit::discard(const VSO &V, + SymbolStringPtr Name) { + assert(Symbols.count(Name) && "Symbol is not part of this MU"); + Symbols.erase(Name); +} + +SymbolFlagsMap +AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { + SymbolFlagsMap Flags; + for (const auto &KV : Symbols) + Flags[KV.first] = KV.second.getFlags(); + return Flags; +} + +ReExportsMaterializationUnit::ReExportsMaterializationUnit( + VSO *SourceVSO, SymbolAliasMap Aliases) + : MaterializationUnit(extractFlags(Aliases)), SourceVSO(SourceVSO), + Aliases(std::move(Aliases)) {} + +void ReExportsMaterializationUnit::materialize( + MaterializationResponsibility R) { + + auto &ES = R.getTargetVSO().getExecutionSession(); + VSO &TgtV = R.getTargetVSO(); + VSO &SrcV = SourceVSO ? *SourceVSO : TgtV; + + // Find the set of requested aliases and aliasees. Return any unrequested + // aliases back to the VSO so as to not prematurely materialize any aliasees. + auto RequestedSymbols = R.getRequestedSymbols(); + SymbolAliasMap RequestedAliases; + + for (auto &Name : RequestedSymbols) { + auto I = Aliases.find(Name); + assert(I != Aliases.end() && "Symbol not found in aliases map?"); + RequestedAliases[Name] = std::move(I->second); + Aliases.erase(I); + } + + if (!Aliases.empty()) { + if (SourceVSO) + R.replace(reexports(*SourceVSO, std::move(Aliases))); + else + R.replace(symbolAliases(std::move(Aliases))); + } + + // The OnResolveInfo struct will hold the aliases and responsibilty for each + // query in the list. + struct OnResolveInfo { + OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases) + : R(std::move(R)), Aliases(std::move(Aliases)) {} + + MaterializationResponsibility R; + SymbolAliasMap Aliases; + }; + + // Build a list of queries to issue. In each round we build the largest set of + // aliases that we can resolve without encountering a chain definition of the + // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would + // be waitin on a symbol that it itself had to resolve. Usually this will just + // involve one round and a single query. + + std::vector<std::pair<SymbolNameSet, std::shared_ptr<OnResolveInfo>>> + QueryInfos; + while (!RequestedAliases.empty()) { + SymbolNameSet ResponsibilitySymbols; + SymbolNameSet QuerySymbols; + SymbolAliasMap QueryAliases; + + for (auto I = RequestedAliases.begin(), E = RequestedAliases.end(); + I != E;) { + auto Tmp = I++; + + // Chain detected. Skip this symbol for this round. + if (&SrcV == &TgtV && (QueryAliases.count(Tmp->second.Aliasee) || + RequestedAliases.count(Tmp->second.Aliasee))) + continue; + + ResponsibilitySymbols.insert(Tmp->first); + QuerySymbols.insert(Tmp->second.Aliasee); + QueryAliases[Tmp->first] = std::move(Tmp->second); + RequestedAliases.erase(Tmp); + } + assert(!QuerySymbols.empty() && "Alias cycle detected!"); + + auto QueryInfo = std::make_shared<OnResolveInfo>( + R.delegate(ResponsibilitySymbols), std::move(QueryAliases)); + QueryInfos.push_back( + make_pair(std::move(QuerySymbols), std::move(QueryInfo))); + } + + // Issue the queries. + while (!QueryInfos.empty()) { + auto QuerySymbols = std::move(QueryInfos.back().first); + auto QueryInfo = std::move(QueryInfos.back().second); + + QueryInfos.pop_back(); + + auto RegisterDependencies = [QueryInfo, + &SrcV](const SymbolDependenceMap &Deps) { + // If there were no materializing symbols, just bail out. + if (Deps.empty()) + return; + + // Otherwise the only deps should be on SrcV. + assert(Deps.size() == 1 && Deps.count(&SrcV) && + "Unexpected dependencies for reexports"); + + auto &SrcVDeps = Deps.find(&SrcV)->second; + SymbolDependenceMap PerAliasDepsMap; + auto &PerAliasDeps = PerAliasDepsMap[&SrcV]; + + for (auto &KV : QueryInfo->Aliases) + if (SrcVDeps.count(KV.second.Aliasee)) { + PerAliasDeps = {KV.second.Aliasee}; + QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); + } + }; + + auto OnResolve = [QueryInfo](Expected<SymbolMap> Result) { + if (Result) { + SymbolMap ResolutionMap; + for (auto &KV : QueryInfo->Aliases) { + assert(Result->count(KV.second.Aliasee) && + "Result map missing entry?"); + ResolutionMap[KV.first] = JITEvaluatedSymbol( + (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + } + QueryInfo->R.resolve(ResolutionMap); + QueryInfo->R.finalize(); + } else { + auto &ES = QueryInfo->R.getTargetVSO().getExecutionSession(); + ES.reportError(Result.takeError()); + QueryInfo->R.failMaterialization(); + } + }; + + auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; + + ES.lookup({&SrcV}, QuerySymbols, std::move(OnResolve), std::move(OnReady), + std::move(RegisterDependencies)); + } +} + +void ReExportsMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { + assert(Aliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + Aliases.erase(Name); +} + +SymbolFlagsMap +ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) + SymbolFlags[KV.first] = KV.second.AliasFlags; + + return SymbolFlags; +} + +Expected<SymbolAliasMap> +buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols) { + auto Flags = SourceV.lookupFlags(Symbols); + + if (Flags.size() != Symbols.size()) { + SymbolNameSet Unresolved = Symbols; + for (auto &KV : Flags) + Unresolved.erase(KV.first); + return make_error<SymbolsNotFound>(std::move(Unresolved)); + } + + SymbolAliasMap Result; + for (auto &Name : Symbols) { + assert(Flags.count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]); + } + + return Result; +} + +Error VSO::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { + return ES.runSessionLocked([&]() -> Error { + std::vector<SymbolMap::iterator> AddedSyms; + + for (auto &KV : SymbolFlags) { + SymbolMap::iterator EntryItr; + bool Added; + + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Materializing; + + std::tie(EntryItr, Added) = Symbols.insert( + std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + + if (Added) + AddedSyms.push_back(EntryItr); + else { + // Remove any symbols already added. + for (auto &SI : AddedSyms) + Symbols.erase(SI); + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(*KV.first); + } + } + + return Error::success(); + }); +} + +void VSO::replace(std::unique_ptr<MaterializationUnit> MU) { + assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + + auto MustRunMU = + ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> { + +#ifndef NDEBUG + for (auto &KV : MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI != Symbols.end() && "Replacing unknown symbol"); + assert(!SymI->second.getFlags().isLazy() && + SymI->second.getFlags().isMaterializing() && + "Can not replace symbol that is not materializing"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Symbol being replaced should have no UnmaterializedInfo"); + } +#endif // NDEBUG + + // If any symbol has pending queries against it then we need to + // materialize MU immediately. + for (auto &KV : MU->getSymbols()) { + auto MII = MaterializingInfos.find(KV.first); + if (MII != MaterializingInfos.end()) { + if (!MII->second.PendingQueries.empty()) + return std::move(MU); + } + } + + // Otherwise, make MU responsible for all the symbols. + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + for (auto &KV : UMI->MU->getSymbols()) { + assert(!KV.second.isLazy() && + "Lazy flag should be managed internally."); + assert(!KV.second.isMaterializing() && + "Materializing flags should be managed internally."); + + auto SymI = Symbols.find(KV.first); + JITSymbolFlags ReplaceFlags = KV.second; + ReplaceFlags |= JITSymbolFlags::Lazy; + SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(), + std::move(ReplaceFlags)); + UnmaterializedInfos[KV.first] = UMI; + } + + return nullptr; + }); + + if (MustRunMU) + ES.dispatchMaterialization(*this, std::move(MustRunMU)); +} + +SymbolNameSet VSO::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) { + return ES.runSessionLocked([&]() { + SymbolNameSet RequestedSymbols; + + for (auto &KV : SymbolFlags) { + assert(Symbols.count(KV.first) && "VSO does not cover this symbol?"); + assert(Symbols[KV.first].getFlags().isMaterializing() && + "getRequestedSymbols can only be called for materializing " + "symbols"); + auto I = MaterializingInfos.find(KV.first); + if (I == MaterializingInfos.end()) + continue; + + if (!I->second.PendingQueries.empty()) + RequestedSymbols.insert(KV.first); + } + + return RequestedSymbols; + }); +} + +void VSO::addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + assert(Symbols.count(Name) && "Name not in symbol table"); + assert((Symbols[Name].getFlags().isLazy() || + Symbols[Name].getFlags().isMaterializing()) && + "Symbol is not lazy or materializing"); + + auto &MI = MaterializingInfos[Name]; + assert(!MI.IsFinalized && "Can not add dependencies to finalized symbol"); + + for (auto &KV : Dependencies) { + assert(KV.first && "Null VSO in dependency?"); + auto &OtherVSO = *KV.first; + auto &DepsOnOtherVSO = MI.UnfinalizedDependencies[&OtherVSO]; + + for (auto &OtherSymbol : KV.second) { +#ifndef NDEBUG + // Assert that this symbol exists and has not been finalized already. + auto SymI = OtherVSO.Symbols.find(OtherSymbol); + assert(SymI != OtherVSO.Symbols.end() && + (SymI->second.getFlags().isLazy() || + SymI->second.getFlags().isMaterializing()) && + "Dependency on finalized symbol"); +#endif + + auto &OtherMI = OtherVSO.MaterializingInfos[OtherSymbol]; + + if (OtherMI.IsFinalized) + transferFinalizedNodeDependencies(MI, Name, OtherMI); + else if (&OtherVSO != this || OtherSymbol != Name) { + OtherMI.Dependants[this].insert(Name); + DepsOnOtherVSO.insert(OtherSymbol); + } + } + + if (DepsOnOtherVSO.empty()) + MI.UnfinalizedDependencies.erase(&OtherVSO); + } +} + +void VSO::resolve(const SymbolMap &Resolved) { + auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet FullyResolvedQueries; + for (const auto &KV : Resolved) { + auto &Name = KV.first; + auto Sym = KV.second; + + assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() && + "Materializing flags should be managed internally"); + + auto I = Symbols.find(Name); + + assert(I != Symbols.end() && "Symbol not found"); + assert(!I->second.getFlags().isLazy() && + I->second.getFlags().isMaterializing() && + "Symbol should be materializing"); + assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); + + assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == + (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) & + ~JITSymbolFlags::Weak) && + "Resolved flags should match the declared flags"); + + // Once resolved, symbols can never be weak. + JITSymbolFlags ResolvedFlags = Sym.getFlags(); + ResolvedFlags &= ~JITSymbolFlags::Weak; + ResolvedFlags |= JITSymbolFlags::Materializing; + I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags); + + auto &MI = MaterializingInfos[Name]; + for (auto &Q : MI.PendingQueries) { + Q->resolve(Name, Sym); + if (Q->isFullyResolved()) + FullyResolvedQueries.insert(Q); + } + } + + return FullyResolvedQueries; + }); + + for (auto &Q : FullyResolvedQueries) { + assert(Q->isFullyResolved() && "Q not fully resolved"); + Q->handleFullyResolved(); + } +} + +void VSO::finalize(const SymbolFlagsMap &Finalized) { + auto FullyReadyQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet ReadyQueries; + + for (const auto &KV : Finalized) { + const auto &Name = KV.first; + + auto MII = MaterializingInfos.find(Name); + assert(MII != MaterializingInfos.end() && + "Missing MaterializingInfo entry"); + + auto &MI = MII->second; + + // For each dependant, transfer this node's unfinalized dependencies to + // it. If the dependant node is fully finalized then notify any pending + // queries. + for (auto &KV : MI.Dependants) { + auto &DependantVSO = *KV.first; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantVSO.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantVSO.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); + + auto &DependantMI = DependantMII->second; + + // Remove the dependant's dependency on this node. + assert(DependantMI.UnfinalizedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); + DependantMI.UnfinalizedDependencies[this].erase(Name); + if (DependantMI.UnfinalizedDependencies[this].empty()) + DependantMI.UnfinalizedDependencies.erase(this); + + // Transfer unfinalized dependencies from this node to the dependant. + DependantVSO.transferFinalizedNodeDependencies(DependantMI, + DependantName, MI); + + // If the dependant is finalized and this node was the last of its + // unfinalized dependencies then notify any pending queries on the + // dependant node. + if (DependantMI.IsFinalized && + DependantMI.UnfinalizedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); + for (auto &Q : DependantMI.PendingQueries) { + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ReadyQueries.insert(Q); + Q->removeQueryDependence(DependantVSO, DependantName); + } + + // If this dependant node was fully finalized we can erase its + // MaterializingInfo and update its materializing state. + assert(DependantVSO.Symbols.count(DependantName) && + "Dependant has no entry in the Symbols table"); + auto &DependantSym = DependantVSO.Symbols[DependantName]; + DependantSym.setFlags(static_cast<JITSymbolFlags::FlagNames>( + DependantSym.getFlags() & ~JITSymbolFlags::Materializing)); + DependantVSO.MaterializingInfos.erase(DependantMII); + } + } + } + MI.Dependants.clear(); + MI.IsFinalized = true; + + if (MI.UnfinalizedDependencies.empty()) { + for (auto &Q : MI.PendingQueries) { + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ReadyQueries.insert(Q); + Q->removeQueryDependence(*this, Name); + } + assert(Symbols.count(Name) && + "Symbol has no entry in the Symbols table"); + auto &Sym = Symbols[Name]; + Sym.setFlags(static_cast<JITSymbolFlags::FlagNames>( + Sym.getFlags() & ~JITSymbolFlags::Materializing)); + MaterializingInfos.erase(MII); + } + } + + return ReadyQueries; + }); + + for (auto &Q : FullyReadyQueries) { + assert(Q->isFullyReady() && "Q is not fully ready"); + Q->handleFullyReady(); + } +} + +void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) { + + // FIXME: This should fail any transitively dependant symbols too. + + auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet FailedQueries; + + for (auto &Name : FailedSymbols) { + auto I = Symbols.find(Name); + assert(I != Symbols.end() && "Symbol not present in this VSO"); + Symbols.erase(I); + + auto MII = MaterializingInfos.find(Name); + + // If we have not created a MaterializingInfo for this symbol yet then + // there is nobody to notify. + if (MII == MaterializingInfos.end()) + continue; + + // Copy all the queries to the FailedQueries list, then abandon them. + // This has to be a copy, and the copy has to come before the abandon + // operation: Each Q.detach() call will reach back into this + // PendingQueries list to remove Q. + for (auto &Q : MII->second.PendingQueries) + FailedQueries.insert(Q); + + for (auto &Q : FailedQueries) + Q->detach(); + + assert(MII->second.PendingQueries.empty() && + "Queries remain after symbol was failed"); + + MaterializingInfos.erase(MII); + } + + return FailedQueries; + }); + + for (auto &Q : FailedQueriesToNotify) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +void VSO::setSearchOrder(VSOList NewSearchOrder, bool SearchThisVSOFirst) { + if (SearchThisVSOFirst && NewSearchOrder.front() != this) + NewSearchOrder.insert(NewSearchOrder.begin(), this); + + ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); +} + +void VSO::addToSearchOrder(VSO &V) { + ES.runSessionLocked([&]() { SearchOrder.push_back(&V); }); +} + +void VSO::replaceInSearchOrder(VSO &OldV, VSO &NewV) { + ES.runSessionLocked([&]() { + auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &OldV); + + if (I != SearchOrder.end()) + *I = &NewV; + }); +} + +void VSO::removeFromSearchOrder(VSO &V) { + ES.runSessionLocked([&]() { + auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &V); + if (I != SearchOrder.end()) + SearchOrder.erase(I); + }); +} + +SymbolFlagsMap VSO::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() { + SymbolFlagsMap Result; + auto Unresolved = lookupFlagsImpl(Result, Names); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, FallbackDefs); + (void)Unresolved2; + assert(Unresolved2.empty() && + "All fallback defs should have been found by lookupFlagsImpl"); + } + }; + return Result; + }); +} + +SymbolNameSet VSO::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { + SymbolNameSet Unresolved; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + + if (I == Symbols.end()) { + Unresolved.insert(Name); + continue; + } + + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags()); + } + + return Unresolved; +} + +void VSO::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, MaterializationUnitList &MUs) { + assert(Q && "Query can not be null"); + + lodgeQueryImpl(Q, Unresolved, MUs); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + for (auto &D : FallbackDefs) + Unresolved.erase(D); + lodgeQueryImpl(Q, FallbackDefs, MUs); + assert(FallbackDefs.empty() && + "All fallback defs should have been found by lookupImpl"); + } + } +} + +void VSO::lodgeQueryImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { + for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { + auto TmpI = I++; + auto Name = *TmpI; + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If we found Name in V, remove it frome the Unresolved set and add it + // to the added set. + Unresolved.erase(TmpI); + + // If the symbol has an address then resolve it. + if (SymI->second.getAddress() != 0) + Q->resolve(Name, SymI->second); + + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.getFlags().isLazy()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + assert(!SymI->second.getFlags().isMaterializing() && + "Materializing and lazy should not both be set"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + auto Flags = SymK->second.getFlags(); + Flags &= ~JITSymbolFlags::Lazy; + Flags |= JITSymbolFlags::Materializing; + SymK->second.setFlags(Flags); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } else if (!SymI->second.getFlags().isMaterializing()) { + // The symbol is neither lazy nor materializing. Finalize it and + // continue. + Q->notifySymbolReady(); + continue; + } + + // Add the query to the PendingQueries list. + assert(SymI->second.getFlags().isMaterializing() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.PendingQueries.push_back(Q); + Q->addQueryDependence(*this, Name); + } +} + +SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { + assert(Q && "Query can not be null"); + + ES.runOutstandingMUs(); + + LookupImplActionFlags ActionFlags = None; + std::vector<std::unique_ptr<MaterializationUnit>> MUs; + + SymbolNameSet Unresolved = std::move(Names); + ES.runSessionLocked([&, this]() { + ActionFlags = lookupImpl(Q, MUs, Unresolved); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + assert(ActionFlags == None && + "ActionFlags set but unresolved symbols remain?"); + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + for (auto &D : FallbackDefs) + Unresolved.erase(D); + ActionFlags = lookupImpl(Q, MUs, FallbackDefs); + assert(FallbackDefs.empty() && + "All fallback defs should have been found by lookupImpl"); + } + } + }); + + assert((MUs.empty() || ActionFlags == None) && + "If action flags are set, there should be no work to do (so no MUs)"); + + if (ActionFlags & NotifyFullyResolved) + Q->handleFullyResolved(); + + if (ActionFlags & NotifyFullyReady) + Q->handleFullyReady(); + + // FIXME: Swap back to the old code below once RuntimeDyld works with + // callbacks from asynchronous queries. + // Add MUs to the OutstandingMUs list. + { + std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); + for (auto &MU : MUs) + ES.OutstandingMUs.push_back(make_pair(this, std::move(MU))); + } + ES.runOutstandingMUs(); + + // Dispatch any required MaterializationUnits for materialization. + // for (auto &MU : MUs) + // ES.dispatchMaterialization(*this, std::move(MU)); + + return Unresolved; +} + +VSO::LookupImplActionFlags +VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { + LookupImplActionFlags ActionFlags = None; + + for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { + auto TmpI = I++; + auto Name = *TmpI; + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If we found Name in V, remove it frome the Unresolved set and add it + // to the dependencies set. + Unresolved.erase(TmpI); + + // If the symbol has an address then resolve it. + if (SymI->second.getAddress() != 0) { + Q->resolve(Name, SymI->second); + if (Q->isFullyResolved()) + ActionFlags |= NotifyFullyResolved; + } + + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.getFlags().isLazy()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + assert(!SymI->second.getFlags().isMaterializing() && + "Materializing and lazy should not both be set"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Kick all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + auto Flags = SymK->second.getFlags(); + Flags &= ~JITSymbolFlags::Lazy; + Flags |= JITSymbolFlags::Materializing; + SymK->second.setFlags(Flags); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } else if (!SymI->second.getFlags().isMaterializing()) { + // The symbol is neither lazy nor materializing. Finalize it and + // continue. + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ActionFlags |= NotifyFullyReady; + continue; + } + + // Add the query to the PendingQueries list. + assert(SymI->second.getFlags().isMaterializing() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.PendingQueries.push_back(Q); + Q->addQueryDependence(*this, Name); + } + + return ActionFlags; +} + +void VSO::dump(raw_ostream &OS) { + ES.runSessionLocked([&, this]() { + OS << "VSO \"" << VSOName + << "\" (ES: " << format("0x%016x", reinterpret_cast<uintptr_t>(&ES)) + << "):\n" + << "Symbol table:\n"; + + for (auto &KV : Symbols) { + OS << " \"" << *KV.first + << "\": " << format("0x%016x", KV.second.getAddress()); + if (KV.second.getFlags().isLazy() || + KV.second.getFlags().isMaterializing()) { + OS << " ("; + if (KV.second.getFlags().isLazy()) { + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << " Lazy (MU=" << I->second->MU.get() << ")"; + } + if (KV.second.getFlags().isMaterializing()) + OS << " Materializing"; + OS << " )\n"; + } else + OS << "\n"; + } + + if (!MaterializingInfos.empty()) + OS << " MaterializingInfos entries:\n"; + for (auto &KV : MaterializingInfos) { + OS << " \"" << *KV.first << "\":\n" + << " IsFinalized = " << (KV.second.IsFinalized ? "true" : "false") + << "\n" + << " " << KV.second.PendingQueries.size() + << " pending queries: { "; + for (auto &Q : KV.second.PendingQueries) + OS << Q.get() << " "; + OS << "}\n Dependants:\n"; + for (auto &KV2 : KV.second.Dependants) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + OS << " Unfinalized Dependencies:\n"; + for (auto &KV2 : KV.second.UnfinalizedDependencies) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + } + }); +} + +VSO::VSO(ExecutionSessionBase &ES, std::string Name) + : ES(ES), VSOName(std::move(Name)) { + SearchOrder.push_back(this); +} + +Error VSO::defineImpl(MaterializationUnit &MU) { + SymbolNameSet Duplicates; + SymbolNameSet MUDefsOverridden; + + struct ExistingDefOverriddenEntry { + SymbolMap::iterator ExistingDefItr; + JITSymbolFlags NewFlags; + }; + std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden; + + for (auto &KV : MU.getSymbols()) { + assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); + assert(!KV.second.isMaterializing() && + "Materializing flags should be managed internally."); + + SymbolMap::iterator EntryItr; + bool Added; + + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Lazy; + + std::tie(EntryItr, Added) = Symbols.insert( + std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + + if (!Added) { + if (KV.second.isStrong()) { + if (EntryItr->second.getFlags().isStrong() || + (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + Duplicates.insert(KV.first); + else + ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + } else + MUDefsOverridden.insert(KV.first); + } + } + + if (!Duplicates.empty()) { + // We need to remove the symbols we added. + for (auto &KV : MU.getSymbols()) { + if (Duplicates.count(KV.first)) + continue; + + bool Found = false; + for (const auto &EDO : ExistingDefsOverridden) + if (EDO.ExistingDefItr->first == KV.first) + Found = true; + + if (!Found) + Symbols.erase(KV.first); + } + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(**Duplicates.begin()); + } + + // Update flags on existing defs and call discard on their materializers. + for (auto &EDO : ExistingDefsOverridden) { + assert(EDO.ExistingDefItr->second.getFlags().isLazy() && + !EDO.ExistingDefItr->second.getFlags().isMaterializing() && + "Overridden existing def should be in the Lazy state"); + + EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + + auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + assert(UMII != UnmaterializedInfos.end() && + "Overridden existing def should have an UnmaterializedInfo"); + + UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + } + + // Discard overridden symbols povided by MU. + for (auto &Sym : MUDefsOverridden) + MU.doDiscard(*this, Sym); + + return Error::success(); +} + +void VSO::detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols) { + for (auto &QuerySymbol : QuerySymbols) { + assert(MaterializingInfos.count(QuerySymbol) && + "QuerySymbol does not have MaterializingInfo"); + auto &MI = MaterializingInfos[QuerySymbol]; + + auto IdenticalQuery = + [&](const std::shared_ptr<AsynchronousSymbolQuery> &R) { + return R.get() == &Q; + }; + + auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(), + IdenticalQuery); + assert(I != MI.PendingQueries.end() && + "Query Q should be in the PendingQueries list for QuerySymbol"); + MI.PendingQueries.erase(I); + } +} + +void VSO::transferFinalizedNodeDependencies( + MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName, + MaterializingInfo &FinalizedMI) { + for (auto &KV : FinalizedMI.UnfinalizedDependencies) { + auto &DependencyVSO = *KV.first; + SymbolNameSet *UnfinalizedDependenciesOnDependencyVSO = nullptr; + + for (auto &DependencyName : KV.second) { + auto &DependencyMI = DependencyVSO.MaterializingInfos[DependencyName]; + + // Do not add self dependencies. + if (&DependencyMI == &DependantMI) + continue; + + // If we haven't looked up the dependencies for DependencyVSO yet, do it + // now and cache the result. + if (!UnfinalizedDependenciesOnDependencyVSO) + UnfinalizedDependenciesOnDependencyVSO = + &DependantMI.UnfinalizedDependencies[&DependencyVSO]; + + DependencyMI.Dependants[this].insert(DependantName); + UnfinalizedDependenciesOnDependencyVSO->insert(DependencyName); + } + } +} + +VSO &ExecutionSession::createVSO(std::string Name) { + return runSessionLocked([&, this]() -> VSO & { + VSOs.push_back(std::unique_ptr<VSO>(new VSO(*this, std::move(Name)))); + return *VSOs.back(); + }); +} + +Expected<SymbolMap> lookup(const VSOList &VSOs, SymbolNameSet Names) { + + if (VSOs.empty()) + return SymbolMap(); + + auto &ES = (*VSOs.begin())->getExecutionSession(); + + return ES.lookup(VSOs, Names, NoDependenciesToRegister, true); +} + +/// Look up a symbol by searching a list of VSOs. +Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name) { + SymbolNameSet Names({Name}); + if (auto ResultMap = lookup(VSOs, std::move(Names))) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +MangleAndInterner::MangleAndInterner(ExecutionSessionBase &ES, + const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.getSymbolStringPool().intern(MangledName); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index b7220dba88e9..6157677ce355 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -13,10 +13,51 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" namespace llvm { namespace orc { +JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) + : TT(std::move(TT)) {} + +Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { + return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); +} + +Expected<std::unique_ptr<TargetMachine>> +JITTargetMachineBuilder::createTargetMachine() { + if (!Arch.empty()) { + Triple::ArchType Type = Triple::getArchTypeForLLVMName(Arch); + + if (Type == Triple::UnknownArch) + return make_error<StringError>(std::string("Unknown arch: ") + Arch, + inconvertibleErrorCode()); + } + + std::string ErrMsg; + auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); + if (!TheTarget) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + auto *TM = + TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), + Options, RM, CM, OptLevel, /*JIT*/ true); + if (!TM) + return make_error<StringError>("Could not allocate target machine", + inconvertibleErrorCode()); + + return std::unique_ptr<TargetMachine>(TM); +} + +JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( + const std::vector<std::string> &FeatureVec) { + for (const auto &F : FeatureVec) + Features.AddFeature(F); + return *this; +} + CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) : InitList( GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), @@ -67,7 +108,9 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const { } ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0)); - Value *Data = CS->getOperand(2); + Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr; + if (Data && !isa<GlobalValue>(Data)) + Data = nullptr; return Element(Priority->getZExtValue(), Func, Data); } @@ -83,20 +126,123 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) { CtorDtorIterator(DtorsList, true)); } -void LocalCXXRuntimeOverrides::runDestructors() { +void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) { + if (CtorDtors.begin() == CtorDtors.end()) + return; + + MangleAndInterner Mangle( + V.getExecutionSession(), + (*CtorDtors.begin()).Func->getParent()->getDataLayout()); + + for (const auto &CtorDtor : CtorDtors) { + assert(CtorDtor.Func && CtorDtor.Func->hasName() && + "Ctor/Dtor function must be named to be runnable under the JIT"); + + if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) { + dbgs() << " Skipping because why now?\n"; + continue; + } + + CtorDtorsByPriority[CtorDtor.Priority].push_back( + Mangle(CtorDtor.Func->getName())); + } +} + +Error CtorDtorRunner2::run() { + using CtorDtorTy = void (*)(); + + SymbolNameSet Names; + + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + auto Added = Names.insert(Name).second; + (void)Added; + assert(Added && "Ctor/Dtor names clashed"); + } + } + + if (auto CtorDtorMap = lookup({&V}, std::move(Names))) { + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + assert(CtorDtorMap->count(Name) && "No entry for Name"); + auto CtorDtor = reinterpret_cast<CtorDtorTy>( + static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + CtorDtor(); + } + } + return Error::success(); + } else + return CtorDtorMap.takeError(); + + CtorDtorsByPriority.clear(); + + return Error::success(); +} + +void LocalCXXRuntimeOverridesBase::runDestructors() { auto& CXXDestructorDataPairs = DSOHandleOverride; for (auto &P : CXXDestructorDataPairs) P.first(P.second); CXXDestructorDataPairs.clear(); } -int LocalCXXRuntimeOverrides::CXAAtExitOverride(DestructorPtr Destructor, - void *Arg, void *DSOHandle) { +int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, + void *Arg, + void *DSOHandle) { auto& CXXDestructorDataPairs = *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle); CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg)); return 0; } +Error LocalCXXRuntimeOverrides2::enable(VSO &V, MangleAndInterner &Mangle) { + SymbolMap RuntimeInterposes( + {{Mangle("__dso_handle"), + JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), + JITSymbolFlags::Exported)}, + {Mangle("__cxa_atexit"), + JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), + JITSymbolFlags::Exported)}}); + + return V.define(absoluteSymbols(std::move(RuntimeInterposes))); +} + +DynamicLibraryFallbackGenerator::DynamicLibraryFallbackGenerator( + sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + : Dylib(std::move(Dylib)), Allow(std::move(Allow)), + GlobalPrefix(DL.getGlobalPrefix()) {} + +SymbolNameSet DynamicLibraryFallbackGenerator:: +operator()(VSO &V, const SymbolNameSet &Names) { + orc::SymbolNameSet Added; + orc::SymbolMap NewSymbols; + + bool HasGlobalPrefix = (GlobalPrefix != '\0'); + + for (auto &Name : Names) { + if (!Allow(Name) || (*Name).empty()) + continue; + + if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) + continue; + + std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { + Added.insert(Name); + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), + JITSymbolFlags::Exported); + } + } + + // Add any new symbols to V. Since the fallback generator is only called for + // symbols that are not already defined, this will never trigger a duplicate + // definition error, so we can wrap this call in a 'cantFail'. + if (!NewSymbols.empty()) + cantFail(V.define(absoluteSymbols(std::move(NewSymbols)))); + + return Added; +} + } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp new file mode 100644 index 000000000000..0c17f9b7ad49 --- /dev/null +++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -0,0 +1,44 @@ +//===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" + +namespace llvm { +namespace orc { + +IRCompileLayer2::IRCompileLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer, + CompileFunction Compile) + : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {} + +void IRCompileLayer2::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + this->NotifyCompiled = std::move(NotifyCompiled); +} + +void IRCompileLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + assert(M && "Module must not be null"); + + if (auto Obj = Compile(*M)) { + { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + if (NotifyCompiled) + NotifyCompiled(K, std::move(M)); + else + M = nullptr; + } + BaseLayer.emit(std::move(R), std::move(K), std::move(*Obj)); + } else { + R.failMaterialization(); + getExecutionSession().reportError(Obj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp new file mode 100644 index 000000000000..4dd3cfdfe387 --- /dev/null +++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -0,0 +1,34 @@ +//===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRTransformLayer2::IRTransformLayer2(ExecutionSession &ES, + IRLayer &BaseLayer, + TransformFunction Transform) + : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void IRTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + assert(M && "Module must not be null"); + + if (auto TransformedMod = Transform(std::move(M))) + BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedMod)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedMod.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 68397beae63a..9ca2c5cb4a55 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -13,38 +13,123 @@ #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> +using namespace llvm; +using namespace llvm::orc; + +namespace { + +class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { +public: + using CompileFunction = JITCompileCallbackManager::CompileFunction; + + CompileCallbackMaterializationUnit(SymbolStringPtr Name, + CompileFunction Compile) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}})), + Name(std::move(Name)), Compile(std::move(Compile)) {} + +private: + void materialize(MaterializationResponsibility R) { + SymbolMap Result; + Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + R.resolve(Result); + R.finalize(); + } + + void discard(const VSO &V, SymbolStringPtr Name) { + llvm_unreachable("Discard should never occur on a LMU?"); + } + + SymbolStringPtr Name; + CompileFunction Compile; +}; + +} // namespace + namespace llvm { namespace orc { void JITCompileCallbackManager::anchor() {} void IndirectStubsManager::anchor() {} +Expected<JITTargetAddress> +JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { + if (auto TrampolineAddr = getAvailableTrampolineAddr()) { + auto CallbackName = ES.getSymbolStringPool().intern( + std::string("cc") + std::to_string(++NextCallbackId)); + + std::lock_guard<std::mutex> Lock(CCMgrMutex); + AddrToSymbol[*TrampolineAddr] = CallbackName; + cantFail(CallbacksVSO.define( + llvm::make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile)))); + return *TrampolineAddr; + } else + return TrampolineAddr.takeError(); +} + +JITTargetAddress JITCompileCallbackManager::executeCompileCallback( + JITTargetAddress TrampolineAddr) { + SymbolStringPtr Name; + + { + std::unique_lock<std::mutex> Lock(CCMgrMutex); + auto I = AddrToSymbol.find(TrampolineAddr); + + // If this address is not associated with a compile callback then report an + // error to the execution session and return ErrorHandlerAddress to the + // callee. + if (I == AddrToSymbol.end()) { + Lock.unlock(); + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "No compile callback for trampoline at " + << format("0x%016x", TrampolineAddr); + } + ES.reportError( + make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + return ErrorHandlerAddress; + } else + Name = I->second; + } + + if (auto Sym = lookup({&CallbacksVSO}, Name)) + return Sym->getAddress(); + else { + // If anything goes wrong materializing Sym then report it to the session + // and return the ErrorHandlerAddress; + ES.reportError(Sym.takeError()); + return ErrorHandlerAddress; + } +} + std::unique_ptr<JITCompileCallbackManager> -createLocalCompileCallbackManager(const Triple &T, +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { switch (T.getArch()) { default: return nullptr; case Triple::aarch64: { typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86: { typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86_64: { if ( T.getOS() == Triple::OSType::Win32 ) { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } else { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } } @@ -54,7 +139,11 @@ createLocalCompileCallbackManager(const Triple &T, std::function<std::unique_ptr<IndirectStubsManager>()> createLocalIndirectStubsManagerBuilder(const Triple &T) { switch (T.getArch()) { - default: return nullptr; + default: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); + }; case Triple::aarch64: return [](){ @@ -176,7 +265,6 @@ void makeAllSymbolsExternallyAccessible(Module &M) { Function* cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap) { - assert(F.getParent() != &Dst && "Can't copy decl over existing function."); Function *NewF = Function::Create(cast<FunctionType>(F.getValueType()), F.getLinkage(), F.getName(), &Dst); @@ -214,7 +302,6 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap) { - assert(GV.getParent() != &Dst && "Can't copy decl over existing global var."); GlobalVariable *NewGV = new GlobalVariable( Dst, GV.getValueType(), GV.isConstant(), GV.getLinkage(), nullptr, GV.getName(), nullptr, @@ -236,8 +323,8 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV, assert(VMap[&OrigGV] == NewGV && "Incorrect global variable mapping in VMap."); assert(NewGV->getParent() != OrigGV.getParent() && - "moveGlobalVariable should only be used to move initializers between " - "modules"); + "moveGlobalVariableInitializer should only be used to move " + "initializers between modules"); NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, nullptr, Materializer)); diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp new file mode 100644 index 000000000000..52ff4efe56b2 --- /dev/null +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -0,0 +1,134 @@ +//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/Mangler.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<LLJIT>> +LLJIT::Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL) { + return std::unique_ptr<LLJIT>( + new LLJIT(std::move(ES), std::move(TM), std::move(DL))); +} + +Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { + auto InternedName = ES->getSymbolStringPool().intern(Name); + SymbolMap Symbols({{InternedName, Sym}}); + return Main.define(absoluteSymbols(std::move(Symbols))); +} + +Error LLJIT::addIRModule(VSO &V, std::unique_ptr<Module> M) { + assert(M && "Can not add null module"); + + if (auto Err = applyDataLayout(*M)) + return Err; + + auto K = ES->allocateVModule(); + return CompileLayer.add(V, K, std::move(M)); +} + +Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(VSO &V, + StringRef Name) { + return llvm::orc::lookup({&V}, ES->getSymbolStringPool().intern(Name)); +} + +LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL) + : ES(std::move(ES)), Main(this->ES->createVSO("main")), TM(std::move(TM)), + DL(std::move(DL)), + ObjLinkingLayer(*this->ES, + [this](VModuleKey K) { return getMemoryManager(K); }), + CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)), + CtorRunner(Main), DtorRunner(Main) {} + +std::shared_ptr<RuntimeDyld::MemoryManager> +LLJIT::getMemoryManager(VModuleKey K) { + return llvm::make_unique<SectionMemoryManager>(); +} + +std::string LLJIT::mangle(StringRef UnmangledName) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL); + } + return MangledName; +} + +Error LLJIT::applyDataLayout(Module &M) { + if (M.getDataLayout().isDefault()) + M.setDataLayout(DL); + + if (M.getDataLayout() != DL) + return make_error<StringError>( + "Added modules have incompatible data layouts", + inconvertibleErrorCode()); + + return Error::success(); +} + +void LLJIT::recordCtorDtors(Module &M) { + CtorRunner.add(getConstructors(M)); + DtorRunner.add(getDestructors(M)); +} + +Expected<std::unique_ptr<LLLazyJIT>> +LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL, + LLVMContext &Ctx) { + const Triple &TT = TM->getTargetTriple(); + + auto CCMgr = createLocalCompileCallbackManager(TT, *ES, 0); + if (!CCMgr) + return make_error<StringError>( + std::string("No callback manager available for ") + TT.str(), + inconvertibleErrorCode()); + + auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); + if (!ISMBuilder) + return make_error<StringError>( + std::string("No indirect stubs manager builder for ") + TT.str(), + inconvertibleErrorCode()); + + return std::unique_ptr<LLLazyJIT>( + new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL), Ctx, + std::move(CCMgr), std::move(ISMBuilder))); +} + +Error LLLazyJIT::addLazyIRModule(VSO &V, std::unique_ptr<Module> M) { + assert(M && "Can not add null module"); + + if (auto Err = applyDataLayout(*M)) + return Err; + + makeAllSymbolsExternallyAccessible(*M); + + recordCtorDtors(*M); + + auto K = ES->allocateVModule(); + return CODLayer.add(V, K, std::move(M)); +} + +LLLazyJIT::LLLazyJIT( + std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, + DataLayout DL, LLVMContext &Ctx, + std::unique_ptr<JITCompileCallbackManager> CCMgr, + std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) + : LLJIT(std::move(ES), std::move(TM), std::move(DL)), + CCMgr(std::move(CCMgr)), TransformLayer(*this->ES, CompileLayer), + CODLayer(*this->ES, TransformLayer, *this->CCMgr, std::move(ISMBuilder), + [&]() -> LLVMContext & { return Ctx; }) {} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/LLVMBuild.txt b/lib/ExecutionEngine/Orc/LLVMBuild.txt index 8f05172e77a4..3f8fe9545fcb 100644 --- a/lib/ExecutionEngine/Orc/LLVMBuild.txt +++ b/lib/ExecutionEngine/Orc/LLVMBuild.txt @@ -1,4 +1,4 @@ -;===- ./lib/ExecutionEngine/MCJIT/LLVMBuild.txt ----------------*- Conf -*--===; +;===- ./lib/ExecutionEngine/Orc/LLVMBuild.txt ----------------*- Conf -*--===; ; ; The LLVM Compiler Infrastructure ; @@ -19,4 +19,4 @@ type = Library name = OrcJIT parent = ExecutionEngine -required_libraries = Core ExecutionEngine Object RuntimeDyld Support TransformUtils +required_libraries = Core ExecutionEngine Object MC RuntimeDyld Support Target TransformUtils diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp new file mode 100644 index 000000000000..b9da3b7fb8d5 --- /dev/null +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -0,0 +1,106 @@ +//===-------------------- Layer.cpp - Layer interfaces --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} +IRLayer::~IRLayer() {} + +Error IRLayer::add(VSO &V, VModuleKey K, std::unique_ptr<Module> M) { + return V.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( + *this, std::move(K), std::move(M))); +} + +IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, + std::unique_ptr<Module> M) + : MaterializationUnit(SymbolFlagsMap()), M(std::move(M)) { + + MangleAndInterner Mangle(ES, this->M->getDataLayout()); + for (auto &G : this->M->global_values()) { + if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && + !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } + } +} + +IRMaterializationUnit::IRMaterializationUnit( + std::unique_ptr<Module> M, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags)), M(std::move(M)), + SymbolToDefinition(std::move(SymbolToDefinition)) {} + +void IRMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && + "Symbol not provided by this MU, or previously discarded"); + assert(!I->second->isDeclaration() && + "Discard should only apply to definitions"); + I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + SymbolToDefinition.erase(I); +} + +BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( + IRLayer &L, VModuleKey K, std::unique_ptr<Module> M) + : IRMaterializationUnit(L.getExecutionSession(), std::move(M)), + L(L), K(std::move(K)) {} + +void BasicIRLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + L.emit(std::move(R), std::move(K), std::move(M)); +} + +ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} + +ObjectLayer::~ObjectLayer() {} + +Error ObjectLayer::add(VSO &V, VModuleKey K, std::unique_ptr<MemoryBuffer> O) { + return V.define(llvm::make_unique<BasicObjectLayerMaterializationUnit>( + *this, std::move(K), std::move(O))); +} + +BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( + ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O) + : MaterializationUnit(SymbolFlagsMap()), L(L), K(std::move(K)), + O(std::move(O)) { + + auto &ES = L.getExecutionSession(); + auto Obj = cantFail( + object::ObjectFile::createObjectFile(this->O->getMemBufferRef())); + + for (auto &Sym : Obj->symbols()) { + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) && + (Sym.getFlags() & object::BasicSymbolRef::SF_Exported)) { + auto InternedName = + ES.getSymbolStringPool().intern(cantFail(Sym.getName())); + SymbolFlags[InternedName] = JITSymbolFlags::fromObjectSymbol(Sym); + } + } +} + +void BasicObjectLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + L.emit(std::move(R), std::move(K), std::move(O)); +} + +void BasicObjectLayerMaterializationUnit::discard(const VSO &V, + SymbolStringPtr Name) { + // FIXME: Support object file level discard. This could be done by building a + // filter to pass to the object layer along with the object itself. +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp new file mode 100644 index 000000000000..18be9a042f7f --- /dev/null +++ b/lib/ExecutionEngine/Orc/Legacy.cpp @@ -0,0 +1,68 @@ +//===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Legacy.h" + +namespace llvm { +namespace orc { + +void SymbolResolver::anchor() {} + +JITSymbolResolverAdapter::JITSymbolResolverAdapter( + ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR) + : ES(ES), R(R), MR(MR) {} + +Expected<JITSymbolResolverAdapter::LookupResult> +JITSymbolResolverAdapter::lookup(const LookupSet &Symbols) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + auto LookupFn = [&, this](std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Unresolved) { + return R.lookup(std::move(Q), std::move(Unresolved)); + }; + + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + if (MR) + MR->addDependenciesForAll(Deps); + }; + + auto InternedResult = + ES.legacyLookup(ES, std::move(LookupFn), std::move(InternedSymbols), + false, RegisterDependencies); + + if (!InternedResult) + return InternedResult.takeError(); + + JITSymbolResolver::LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = KV.second; + + return Result; +} + +Expected<JITSymbolResolverAdapter::LookupFlagsResult> +JITSymbolResolverAdapter::lookupFlags(const LookupSet &Symbols) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + SymbolFlagsMap SymbolFlags = R.lookupFlags(InternedSymbols); + LookupFlagsResult Result; + for (auto &KV : SymbolFlags) { + ResolvedStrings.insert(KV.first); + Result[*KV.first] = KV.second; + } + + return Result; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp index 8f2d6fd6c32b..3796e3d37bc2 100644 --- a/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -14,11 +14,23 @@ namespace llvm { namespace orc { -JITSymbol NullResolver::findSymbol(const std::string &Name) { +SymbolFlagsMap NullResolver::lookupFlags(const SymbolNameSet &Symbols) { + return SymbolFlagsMap(); +} + +SymbolNameSet +NullResolver::lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) { + assert(Symbols.empty() && "Null resolver: Symbols must be empty"); + return Symbols; +} + +JITSymbol NullLegacyResolver::findSymbol(const std::string &Name) { llvm_unreachable("Unexpected cross-object symbol reference"); } -JITSymbol NullResolver::findSymbolInLogicalDylib(const std::string &Name) { +JITSymbol +NullLegacyResolver::findSymbolInLogicalDylib(const std::string &Name) { llvm_unreachable("Unexpected cross-object symbol reference"); } diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp new file mode 100644 index 000000000000..6980c8140fd0 --- /dev/null +++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -0,0 +1,34 @@ +//===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +ObjectTransformLayer2::ObjectTransformLayer2(ExecutionSession &ES, + ObjectLayer &BaseLayer, + TransformFunction Transform) + : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void ObjectTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Module must not be null"); + + if (auto TransformedObj = Transform(std::move(O))) + BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedObj)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedObj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp index f945acaf95ee..d6005d24a648 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -9,28 +9,20 @@ #include "OrcCBindingsStack.h" #include "llvm-c/OrcBindings.h" +#include "llvm/ExecutionEngine/JITEventListener.h" using namespace llvm; -LLVMSharedModuleRef LLVMOrcMakeSharedModule(LLVMModuleRef Mod) { - return wrap(new std::shared_ptr<Module>(unwrap(Mod))); -} - -void LLVMOrcDisposeSharedModuleRef(LLVMSharedModuleRef SharedMod) { - delete unwrap(SharedMod); -} - LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { TargetMachine *TM2(unwrap(TM)); Triple T(TM2->getTargetTriple()); - auto CompileCallbackMgr = orc::createLocalCompileCallbackManager(T, 0); auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(T); - OrcCBindingsStack *JITStack = new OrcCBindingsStack( - *TM2, std::move(CompileCallbackMgr), IndirectStubsMgrBuilder); + OrcCBindingsStack *JITStack = + new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder)); return wrap(JITStack); } @@ -75,24 +67,24 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, LLVMOrcErrorCode LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMSharedModuleRef Mod, + LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - std::shared_ptr<Module> *M(unwrap(Mod)); - return J.addIRModuleEager(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); + std::unique_ptr<Module> M(unwrap(Mod)); + return J.addIRModuleEager(*RetHandle, std::move(M), SymbolResolver, + SymbolResolverCtx); } LLVMOrcErrorCode LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMSharedModuleRef Mod, + LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - std::shared_ptr<Module> *M(unwrap(Mod)); - return J.addIRModuleLazy(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); + std::unique_ptr<Module> M(unwrap(Mod)); + return J.addIRModuleLazy(*RetHandle, std::move(M), SymbolResolver, + SymbolResolverCtx); } LLVMOrcErrorCode @@ -120,9 +112,27 @@ LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, return J.findSymbolAddress(*RetAddr, SymbolName, true); } +LLVMOrcErrorCode LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + LLVMOrcModuleHandle H, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + return J.findSymbolAddressIn(*RetAddr, H, SymbolName, true); +} + LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { auto *J = unwrap(JITStack); auto Err = J->shutdown(); delete J; return Err; } + +void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->RegisterJITEventListener(unwrap(L)); +} + +void LLVMOrcUnregisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->UnregisterJITEventListener(unwrap(L)); +} diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 405970e063d8..b9f8a370d2f0 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -15,6 +15,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" @@ -33,6 +34,7 @@ #include <algorithm> #include <cstdint> #include <functional> +#include <map> #include <memory> #include <set> #include <string> @@ -42,68 +44,61 @@ namespace llvm { class OrcCBindingsStack; -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(std::shared_ptr<Module>, - LLVMSharedModuleRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) namespace detail { +// FIXME: Kill this off once the Layer concept becomes an interface. +class GenericLayer { +public: + virtual ~GenericLayer() = default; - class GenericHandle { - public: - virtual ~GenericHandle() = default; - - virtual JITSymbol findSymbolIn(const std::string &Name, - bool ExportedSymbolsOnly) = 0; - virtual Error removeModule() = 0; + virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) = 0; + virtual Error removeModule(orc::VModuleKey K) = 0; }; - template <typename LayerT> class GenericHandleImpl : public GenericHandle { + template <typename LayerT> class GenericLayerImpl : public GenericLayer { public: - GenericHandleImpl(LayerT &Layer, typename LayerT::ModuleHandleT Handle) - : Layer(Layer), Handle(std::move(Handle)) {} + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - JITSymbol findSymbolIn(const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - Error removeModule() override { return Layer.removeModule(Handle); } + Error removeModule(orc::VModuleKey K) override { + return Layer.removeModule(K); + } private: LayerT &Layer; - typename LayerT::ModuleHandleT Handle; }; template <> - class GenericHandleImpl<orc::RTDyldObjectLinkingLayer> - : public GenericHandle { + class GenericLayerImpl<orc::RTDyldObjectLinkingLayer> : public GenericLayer { private: using LayerT = orc::RTDyldObjectLinkingLayer; public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - GenericHandleImpl(LayerT &Layer, typename LayerT::ObjHandleT Handle) - : Layer(Layer), Handle(std::move(Handle)) {} - - JITSymbol findSymbolIn(const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - Error removeModule() override { return Layer.removeObject(Handle); } + Error removeModule(orc::VModuleKey K) override { + return Layer.removeObject(K); + } private: LayerT &Layer; - typename LayerT::ObjHandleT Handle; }; - - template <typename LayerT, typename HandleT> - std::unique_ptr<GenericHandleImpl<LayerT>> - createGenericHandle(LayerT &Layer, HandleT Handle) { - return llvm::make_unique<GenericHandleImpl<LayerT>>(Layer, - std::move(Handle)); + template <typename LayerT> + std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { + return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer); } } // end namespace detail @@ -126,20 +121,123 @@ private: using OwningObject = object::OwningBinary<object::ObjectFile>; -public: - using ModuleHandleT = unsigned; + class CBindingsResolver : public orc::SymbolResolver { + public: + CBindingsResolver(OrcCBindingsStack &Stack, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) + : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), + ExternalResolverCtx(std::move(ExternalResolverCtx)) {} + + orc::SymbolFlagsMap + lookupFlags(const orc::SymbolNameSet &Symbols) override { + orc::SymbolFlagsMap SymbolFlags; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) + SymbolFlags[S] = Sym.getFlags(); + else if (auto Err = Sym.takeError()) { + Stack.reportError(std::move(Err)); + return orc::SymbolFlagsMap(); + } + } + + return SymbolFlags; + } + + orc::SymbolNameSet + lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query, + orc::SymbolNameSet Symbols) override { + orc::SymbolNameSet UnresolvedSymbols; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + Query->notifySymbolReady(); + } else { + Stack.ES.legacyFailQuery(*Query, Addr.takeError()); + return orc::SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + Stack.ES.legacyFailQuery(*Query, std::move(Err)); + return orc::SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + + if (Query->isFullyResolved()) + Query->handleFullyResolved(); + + if (Query->isFullyReady()) + Query->handleFullyReady(); + + return UnresolvedSymbols; + } + + private: + JITSymbol findSymbol(const std::string &Name) { + // Search order: + // 1. JIT'd symbols. + // 2. Runtime overrides. + // 3. External resolver (if present). + + if (auto Sym = Stack.CODLayer.findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) + return Sym; + + if (ExternalResolver) + return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx), + JITSymbolFlags::Exported); + + return JITSymbol(nullptr); + } + + OrcCBindingsStack &Stack; + LLVMOrcSymbolResolverFn ExternalResolver; + void *ExternalResolverCtx = nullptr; + }; + +public: OrcCBindingsStack(TargetMachine &TM, - std::unique_ptr<CompileCallbackMgr> CCMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), - CCMgr(std::move(CCMgr)), - ObjectLayer( - []() { - return std::make_shared<SectionMemoryManager>(); - }), + : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)), + DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), + ObjectLayer(ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), - CODLayer(CompileLayer, + CODLayer(ES, CompileLayer, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + return ResolverI->second; + }, + [this](orc::VModuleKey K, + std::shared_ptr<orc::SymbolResolver> Resolver) { + assert(!Resolvers.count(K) && "Resolver already present"); + Resolvers[K] = std::move(Resolver); + }, [](Function &F) { return std::set<Function *>({&F}); }, *this->CCMgr, std::move(IndirectStubsMgrBuilder), false), CXXRuntimeOverrides( @@ -174,15 +272,15 @@ public: createLazyCompileCallback(JITTargetAddress &RetAddr, LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { - if (auto CCInfoOrErr = CCMgr->getCompileCallback()) { - auto &CCInfo = *CCInfoOrErr; - CCInfo.setCompileAction([=]() -> JITTargetAddress { - return Callback(wrap(this), CallbackCtx); - }); - RetAddr = CCInfo.getAddress(); + auto WrappedCallback = [=]() -> JITTargetAddress { + return Callback(wrap(this), CallbackCtx); + }; + + if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) { + RetAddr = *CCAddr; return LLVMOrcErrSuccess; } else - return mapError(CCInfoOrErr.takeError()); + return mapError(CCAddr.takeError()); } LLVMOrcErrorCode createIndirectStub(StringRef StubName, @@ -195,42 +293,9 @@ public: JITTargetAddress Addr) { return mapError(IndirectStubsMgr->updatePointer(Name, Addr)); } - - std::shared_ptr<JITSymbolResolver> - createResolver(LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return orc::createLambdaResolver( - [this, ExternalResolver, ExternalResolverCtx](const std::string &Name) - -> JITSymbol { - // Search order: - // 1. JIT'd symbols. - // 2. Runtime overrides. - // 3. External resolver (if present). - - if (auto Sym = CODLayer.findSymbol(Name, true)) - return Sym; - else if (auto Err = Sym.takeError()) - return Sym.takeError(); - - if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name)) - return Sym; - - if (ExternalResolver) - return JITSymbol( - ExternalResolver(Name.c_str(), ExternalResolverCtx), - JITSymbolFlags::Exported); - - return JITSymbol(nullptr); - }, - [](const std::string &Name) -> JITSymbol { - return JITSymbol(nullptr); - }); - } - template <typename LayerT> LLVMOrcErrorCode - addIRModule(ModuleHandleT &RetHandle, LayerT &Layer, - std::shared_ptr<Module> M, + addIRModule(orc::VModuleKey &RetKey, LayerT &Layer, std::unique_ptr<Module> M, std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { @@ -247,79 +312,73 @@ public: for (auto Dtor : orc::getDestructors(*M)) DtorNames.push_back(mangle(Dtor.Func->getName())); - // Create the resolver. - auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); - // Add the module to the JIT. - ModuleHandleT H; - if (auto LHOrErr = Layer.addModule(std::move(M), std::move(Resolver))) - H = createHandle(Layer, *LHOrErr); - else - return mapError(LHOrErr.takeError()); + RetKey = ES.allocateVModule(); + Resolvers[RetKey] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); + if (auto Err = Layer.addModule(RetKey, std::move(M))) + return mapError(std::move(Err)); + + KeyLayers[RetKey] = detail::createGenericLayer(Layer); // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), H); + orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), + RetKey); if (auto Err = CtorRunner.runViaLayer(*this)) return mapError(std::move(Err)); - IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H); + IRStaticDestructorRunners.emplace_back(std::move(DtorNames), RetKey); - RetHandle = H; return LLVMOrcErrSuccess; } - LLVMOrcErrorCode addIRModuleEager(ModuleHandleT &RetHandle, - std::shared_ptr<Module> M, + LLVMOrcErrorCode addIRModuleEager(orc::VModuleKey &RetKey, + std::unique_ptr<Module> M, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - return addIRModule(RetHandle, CompileLayer, std::move(M), + return addIRModule(RetKey, CompileLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode addIRModuleLazy(ModuleHandleT &RetHandle, - std::shared_ptr<Module> M, + LLVMOrcErrorCode addIRModuleLazy(orc::VModuleKey &RetKey, + std::unique_ptr<Module> M, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - return addIRModule(RetHandle, CODLayer, std::move(M), + return addIRModule(RetKey, CODLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode removeModule(ModuleHandleT H) { - if (auto Err = GenericHandles[H]->removeModule()) + LLVMOrcErrorCode removeModule(orc::VModuleKey K) { + // FIXME: Should error release the module key? + if (auto Err = KeyLayers[K]->removeModule(K)) return mapError(std::move(Err)); - GenericHandles[H] = nullptr; - FreeHandleIndexes.push_back(H); + ES.releaseVModule(K); + KeyLayers.erase(K); return LLVMOrcErrSuccess; } - LLVMOrcErrorCode addObject(ModuleHandleT &RetHandle, + LLVMOrcErrorCode addObject(orc::VModuleKey &RetKey, std::unique_ptr<MemoryBuffer> ObjBuffer, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - if (auto ObjOrErr = - object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())) { - auto &Obj = *ObjOrErr; - auto OwningObj = - std::make_shared<OwningObject>(std::move(Obj), std::move(ObjBuffer)); + if (auto Obj = object::ObjectFile::createObjectFile( + ObjBuffer->getMemBufferRef())) { - // Create the resolver. - auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); + RetKey = ES.allocateVModule(); + Resolvers[RetKey] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); - ModuleHandleT H; - if (auto HOrErr = ObjectLayer.addObject(std::move(OwningObj), - std::move(Resolver))) - H = createHandle(ObjectLayer, *HOrErr); - else - return mapError(HOrErr.takeError()); + if (auto Err = ObjectLayer.addObject(RetKey, std::move(ObjBuffer))) + return mapError(std::move(Err)); - RetHandle = H; + KeyLayers[RetKey] = detail::createGenericLayer(ObjectLayer); return LLVMOrcErrSuccess; } else - return mapError(ObjOrErr.takeError()); + return mapError(Obj.takeError()); } JITSymbol findSymbol(const std::string &Name, @@ -329,9 +388,10 @@ public: return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); } - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return GenericHandles[H]->findSymbolIn(Name, ExportedSymbolsOnly); + assert(KeyLayers.count(K) && "looking up symbol in unknown module"); + return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); } LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr, @@ -354,26 +414,48 @@ public: return LLVMOrcErrSuccess; } + LLVMOrcErrorCode findSymbolAddressIn(JITTargetAddress &RetAddr, + orc::VModuleKey K, + const std::string &Name, + bool ExportedSymbolsOnly) { + RetAddr = 0; + if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) { + RetAddr = *AddrOrErr; + return LLVMOrcErrSuccess; + } else + return mapError(AddrOrErr.takeError()); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return mapError(std::move(Err)); + } + // Otherwise we had a successful lookup but got a null result. We already + // set RetAddr to '0' above, so just return success. + return LLVMOrcErrSuccess; + } + const std::string &getErrorMessage() const { return ErrMsg; } -private: - template <typename LayerT, typename HandleT> - unsigned createHandle(LayerT &Layer, HandleT Handle) { - unsigned NewHandle; - if (!FreeHandleIndexes.empty()) { - NewHandle = FreeHandleIndexes.back(); - FreeHandleIndexes.pop_back(); - GenericHandles[NewHandle] = - detail::createGenericHandle(Layer, std::move(Handle)); - return NewHandle; - } else { - NewHandle = GenericHandles.size(); - GenericHandles.push_back( - detail::createGenericHandle(Layer, std::move(Handle))); + void RegisterJITEventListener(JITEventListener *L) { + if (!L) + return; + EventListeners.push_back(L); + } + + void UnregisterJITEventListener(JITEventListener *L) { + if (!L) + return; + + auto I = find(reverse(EventListeners), L); + if (I != EventListeners.rend()) { + std::swap(*I, EventListeners.back()); + EventListeners.pop_back(); } - return NewHandle; } +private: + LLVMOrcErrorCode mapError(Error Err) { LLVMOrcErrorCode Result = LLVMOrcErrSuccess; handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { @@ -386,22 +468,44 @@ private: return Result; } + void reportError(Error Err) { + // FIXME: Report errors on the execution session. + logAllUnhandledErrors(std::move(Err), errs(), "ORC error: "); + }; + + void notifyFinalized(orc::VModuleKey K, + const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + for (auto &Listener : EventListeners) + Listener->NotifyObjectEmitted(Obj, LoadedObjInfo); + } + + void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { + for (auto &Listener : EventListeners) + Listener->NotifyFreeingObject(Obj); + } + + orc::ExecutionSession ES; + std::unique_ptr<CompileCallbackMgr> CCMgr; + + std::vector<JITEventListener *> EventListeners; + DataLayout DL; SectionMemoryManager CCMgrMemMgr; std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; - std::unique_ptr<CompileCallbackMgr> CCMgr; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; CODLayerT CODLayer; - std::vector<std::unique_ptr<detail::GenericHandle>> GenericHandles; - std::vector<unsigned> FreeHandleIndexes; + std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; std::string ErrMsg; + + std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers; }; } // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/OrcError.cpp b/lib/ExecutionEngine/Orc/OrcError.cpp index c218cb9a523c..f4102b359a6b 100644 --- a/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/lib/ExecutionEngine/Orc/OrcError.cpp @@ -29,6 +29,12 @@ public: std::string message(int condition) const override { switch (static_cast<OrcErrorCode>(condition)) { + case OrcErrorCode::UnknownORCError: + return "Unknown ORC error"; + case OrcErrorCode::DuplicateDefinition: + return "Duplicate symbol definition"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; case OrcErrorCode::RemoteAllocatorDoesNotExist: return "Remote allocator does not exist"; case OrcErrorCode::RemoteAllocatorIdAlreadyInUse: @@ -45,8 +51,6 @@ public: return "Could not negotiate RPC function"; case OrcErrorCode::RPCResponseAbandoned: return "RPC response abandoned"; - case OrcErrorCode::JITSymbolNotFound: - return "JIT symbol not found"; case OrcErrorCode::UnexpectedRPCCall: return "Unexpected RPC call"; case OrcErrorCode::UnexpectedRPCResponse: @@ -67,6 +71,7 @@ static ManagedStatic<OrcErrorCategory> OrcErrCat; namespace llvm { namespace orc { +char DuplicateDefinition::ID = 0; char JITSymbolNotFound::ID = 0; std::error_code orcError(OrcErrorCode ErrCode) { @@ -74,6 +79,22 @@ std::error_code orcError(OrcErrorCode ErrCode) { return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat); } + +DuplicateDefinition::DuplicateDefinition(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code DuplicateDefinition::convertToErrorCode() const { + return orcError(OrcErrorCode::DuplicateDefinition); +} + +void DuplicateDefinition::log(raw_ostream &OS) const { + OS << "Duplicate definition of symbol '" << SymbolName << "'"; +} + +const std::string &DuplicateDefinition::getSymbolName() const { + return SymbolName; +} + JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) : SymbolName(std::move(SymbolName)) {} diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index f89f21adff41..4def579e7097 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -125,8 +125,13 @@ OrcMCJITReplacement::runFunction(Function *F, } void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { - for (auto &M : LocalModules) - ExecutionEngine::runStaticConstructorsDestructors(*M, isDtors); + auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; + + for (auto &KV : CtorDtorsMap) + cantFail(CtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) + .runViaLayer(LazyEmitLayer)); + + CtorDtorsMap.clear(); } } // End namespace orc. diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 1dc8d4ac7bc5..abe89ce70af9 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" @@ -54,6 +55,7 @@ class ObjectCache; namespace orc { class OrcMCJITReplacement : public ExecutionEngine { + // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are // expecting - see finalizeMemory. @@ -138,18 +140,75 @@ class OrcMCJITReplacement : public ExecutionEngine { std::shared_ptr<MCJITMemoryManager> ClientMM; }; - class LinkingResolver : public JITSymbolResolver { + class LinkingORCResolver : public orc::SymbolResolver { public: - LinkingResolver(OrcMCJITReplacement &M) : M(M) {} + LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} + + SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) override { + SymbolFlagsMap SymbolFlags; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + SymbolFlags[S] = Sym.getFlags(); + } else if (auto Err = Sym.takeError()) { + M.reportError(std::move(Err)); + return SymbolFlagsMap(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) { + SymbolFlags[S] = Sym2.getFlags(); + } else if (auto Err = Sym2.takeError()) { + M.reportError(std::move(Err)); + return SymbolFlagsMap(); + } + } + } - JITSymbol findSymbol(const std::string &Name) override { - return M.ClientResolver->findSymbol(Name); + return SymbolFlags; } - JITSymbol findSymbolInLogicalDylib(const std::string &Name) override { - if (auto Sym = M.findMangledSymbol(Name)) - return Sym; - return M.ClientResolver->findSymbolInLogicalDylib(Name); + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) override { + SymbolNameSet UnresolvedSymbols; + bool NewSymbolsResolved = false; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + Query->notifySymbolReady(); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { + if (auto Addr = Sym2.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); + Query->notifySymbolReady(); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym2.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + } + + if (NewSymbolsResolved && Query->isFullyResolved()) + Query->handleFullyResolved(); + + if (NewSymbolsResolved && Query->isFullyReady()) + Query->handleFullyReady(); + + return UnresolvedSymbols; } private: @@ -160,26 +219,37 @@ private: static ExecutionEngine * createOrcMCJITReplacement(std::string *ErrorMsg, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, std::unique_ptr<TargetMachine> TM) { return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), std::move(TM)); } + void reportError(Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: "); + } + public: - OrcMCJITReplacement( - std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> ClientResolver, - std::unique_ptr<TargetMachine> TM) - : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), - MemMgr(std::make_shared<MCJITReplacementMemMgr>(*this, - std::move(MemMgr))), - Resolver(std::make_shared<LinkingResolver>(*this)), + OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, + std::unique_ptr<TargetMachine> TM) + : ExecutionEngine(TM->createDataLayout()), + TM(std::move(TM)), + MemMgr( + std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), + Resolver(std::make_shared<LinkingORCResolver>(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), - ObjectLayer([this]() { return this->MemMgr; }, NotifyObjectLoaded, - NotifyFinalized), - CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)), + ObjectLayer( + ES, + [this](VModuleKey K) { + return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; + }, + NotifyObjectLoaded, NotifyFinalized), + CompileLayer(ObjectLayer, SimpleCompiler(*this->TM), + [this](VModuleKey K, std::unique_ptr<Module> M) { + Modules.push_back(std::move(M)); + }), LazyEmitLayer(CompileLayer) {} static void Register() { @@ -194,43 +264,63 @@ public: } else { assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); } - auto *MPtr = M.release(); - ShouldDelete[MPtr] = true; - auto Deleter = [this](Module *Mod) { - auto I = ShouldDelete.find(Mod); - if (I != ShouldDelete.end() && I->second) - delete Mod; - }; - LocalModules.push_back(std::shared_ptr<Module>(MPtr, std::move(Deleter))); - cantFail(LazyEmitLayer.addModule(LocalModules.back(), Resolver)); + + // Rename, bump linkage and record static constructors and destructors. + // We have to do this before we hand over ownership of the module to the + // JIT. + std::vector<std::string> CtorNames, DtorNames; + { + unsigned CtorId = 0, DtorId = 0; + for (auto Ctor : orc::getConstructors(*M)) { + std::string NewCtorName = ("$static_ctor." + Twine(CtorId++)).str(); + Ctor.Func->setName(NewCtorName); + Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); + Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); + CtorNames.push_back(mangle(NewCtorName)); + } + for (auto Dtor : orc::getDestructors(*M)) { + std::string NewDtorName = ("$static_dtor." + Twine(DtorId++)).str(); + dbgs() << "Found dtor: " << NewDtorName << "\n"; + Dtor.Func->setName(NewDtorName); + Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); + Dtor.Func->setVisibility(GlobalValue::HiddenVisibility); + DtorNames.push_back(mangle(NewDtorName)); + } + } + + auto K = ES.allocateVModule(); + + UnexecutedConstructors[K] = std::move(CtorNames); + UnexecutedDestructors[K] = std::move(DtorNames); + + cantFail(LazyEmitLayer.addModule(K, std::move(M))); } void addObjectFile(std::unique_ptr<object::ObjectFile> O) override { - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O), - nullptr); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData()))); } void addObjectFile(object::OwningBinary<object::ObjectFile> O) override { - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O)); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + std::unique_ptr<object::ObjectFile> Obj; + std::unique_ptr<MemoryBuffer> ObjBuffer; + std::tie(Obj, ObjBuffer) = O.takeBinary(); + cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer))); } void addArchive(object::OwningBinary<object::Archive> A) override { Archives.push_back(std::move(A)); } - + bool removeModule(Module *M) override { - for (auto I = LocalModules.begin(), E = LocalModules.end(); I != E; ++I) { - if (I->get() == M) { - ShouldDelete[M] = false; - LocalModules.erase(I); - return true; - } - } - return false; + auto I = Modules.begin(); + for (auto E = Modules.end(); I != E; ++I) + if (I->get() == M) + break; + if (I == Modules.end()) + return false; + Modules.erase(I); + return true; } uint64_t getSymbolAddress(StringRef Name) { @@ -238,7 +328,7 @@ public: } JITSymbol findSymbol(StringRef Name) { - return findMangledSymbol(Mangle(Name)); + return findMangledSymbol(mangle(Name)); } void finalizeObject() override { @@ -318,12 +408,9 @@ private: } std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); if (ChildBin->isObject()) { - std::unique_ptr<object::ObjectFile> ChildObj( - static_cast<object::ObjectFile*>(ChildBinOrErr->release())); - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>( - std::move(ChildObj), nullptr); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), + MemoryBuffer::getMemBufferCopy(ChildBin->getData()))); if (auto Sym = ObjectLayer.findSymbol(Name, true)) return Sym; } @@ -339,12 +426,11 @@ private: NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {} - void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H, - const RTDyldObjectLinkingLayer::ObjectPtr &Obj, + void operator()(VModuleKey K, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &Info) const { - M.UnfinalizedSections[H] = std::move(M.SectionsAllocatedSinceLastLoad); + M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad); M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); - M.MemMgr->notifyObjectLoaded(&M, *Obj->getBinary()); + M.MemMgr->notifyObjectLoaded(&M, Obj); } private: OrcMCJITReplacement &M; @@ -354,15 +440,16 @@ private: public: NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {} - void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H) { - M.UnfinalizedSections.erase(H); + void operator()(VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &Info) { + M.UnfinalizedSections.erase(K); } private: OrcMCJITReplacement &M; }; - std::string Mangle(StringRef Name) { + std::string mangle(StringRef Name) { std::string MangledName; { raw_string_ostream MangledNameStream(MangledName); @@ -375,17 +462,18 @@ private: using CompileLayerT = IRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; + ExecutionSession ES; + std::unique_ptr<TargetMachine> TM; std::shared_ptr<MCJITReplacementMemMgr> MemMgr; - std::shared_ptr<LinkingResolver> Resolver; - std::shared_ptr<JITSymbolResolver> ClientResolver; + std::shared_ptr<LinkingORCResolver> Resolver; + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; Mangler Mang; // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr // delete blocks in LocalModules refer to the ShouldDelete map, so // LocalModules needs to be destructed before ShouldDelete. std::map<Module*, bool> ShouldDelete; - std::vector<std::shared_ptr<Module>> LocalModules; NotifyObjectLoadedT NotifyObjectLoaded; NotifyFinalizedT NotifyFinalized; @@ -394,19 +482,15 @@ private: CompileLayerT CompileLayer; LazyEmitLayerT LazyEmitLayer; + std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors; + std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors; + // We need to store ObjLayerT::ObjSetHandles for each of the object sets // that have been emitted but not yet finalized so that we can forward the // mapSectionAddress calls appropriately. using SectionAddrSet = std::set<const void *>; - struct ObjHandleCompare { - bool operator()(ObjectLayerT::ObjHandleT H1, - ObjectLayerT::ObjHandleT H2) const { - return &*H1 < &*H2; - } - }; SectionAddrSet SectionsAllocatedSinceLastLoad; - std::map<ObjectLayerT::ObjHandleT, SectionAddrSet, ObjHandleCompare> - UnfinalizedSections; + std::map<VModuleKey, SectionAddrSet> UnfinalizedSections; std::vector<object::OwningBinary<object::Archive>> Archives; }; diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp new file mode 100644 index 000000000000..71b4b73ca6d3 --- /dev/null +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -0,0 +1,177 @@ +//===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" + +namespace { + +using namespace llvm; +using namespace llvm::orc; + +class VSOSearchOrderResolver : public JITSymbolResolver { +public: + VSOSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} + + Expected<LookupResult> lookup(const LookupSet &Symbols) { + auto &ES = MR.getTargetVSO().getExecutionSession(); + SymbolNameSet InternedSymbols; + + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + MR.addDependenciesForAll(Deps); + }; + + auto InternedResult = + MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { + return ES.lookup(VSOs, InternedSymbols, RegisterDependencies, false); + }); + + if (!InternedResult) + return InternedResult.takeError(); + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + + return Result; + } + + Expected<LookupFlagsResult> lookupFlags(const LookupSet &Symbols) { + auto &ES = MR.getTargetVSO().getExecutionSession(); + + SymbolNameSet InternedSymbols; + + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + SymbolFlagsMap InternedResult; + MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { + // An empty search order is pathalogical, but allowed. + if (VSOs.empty()) + return; + + assert(VSOs.front() && "VSOList entry can not be null"); + InternedResult = VSOs.front()->lookupFlags(InternedSymbols); + }); + + LookupFlagsResult Result; + for (auto &KV : InternedResult) + Result[*KV.first] = std::move(KV.second); + + return Result; + } + +private: + MaterializationResponsibility &MR; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +RTDyldObjectLinkingLayer2::RTDyldObjectLinkingLayer2( + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, + NotifyLoadedFunction NotifyLoaded, NotifyFinalizedFunction NotifyFinalized) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), ProcessAllSections(false) {} + +void RTDyldObjectLinkingLayer2::emit(MaterializationResponsibility R, + VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + + auto &ES = getExecutionSession(); + + auto ObjFile = object::ObjectFile::createObjectFile(*O); + if (!ObjFile) { + getExecutionSession().reportError(ObjFile.takeError()); + R.failMaterialization(); + } + + auto MemoryManager = GetMemoryManager(K); + + VSOSearchOrderResolver Resolver(R); + auto RTDyld = llvm::make_unique<RuntimeDyld>(*MemoryManager, Resolver); + RTDyld->setProcessAllSections(ProcessAllSections); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + + assert(!ActiveRTDylds.count(K) && + "An active RTDyld already exists for this key?"); + ActiveRTDylds[K] = RTDyld.get(); + + assert(!MemMgrs.count(K) && + "A memory manager already exists for this key?"); + MemMgrs[K] = std::move(MemoryManager); + } + + auto Info = RTDyld->loadObject(**ObjFile); + + { + std::set<StringRef> InternalSymbols; + for (auto &Sym : (*ObjFile)->symbols()) { + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) { + if (auto SymName = Sym.getName()) + InternalSymbols.insert(*SymName); + else { + ES.reportError(SymName.takeError()); + R.failMaterialization(); + return; + } + } + } + + SymbolMap Symbols; + for (auto &KV : RTDyld->getSymbolTable()) + if (!InternalSymbols.count(KV.first)) + Symbols[ES.getSymbolStringPool().intern(KV.first)] = KV.second; + + R.resolve(Symbols); + } + + if (NotifyLoaded) + NotifyLoaded(K, **ObjFile, *Info); + + RTDyld->finalizeWithMemoryManagerLocking(); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + ActiveRTDylds.erase(K); + } + + if (RTDyld->hasError()) { + ES.reportError(make_error<StringError>(RTDyld->getErrorString(), + inconvertibleErrorCode())); + R.failMaterialization(); + return; + } + + R.finalize(); + + if (NotifyFinalized) + NotifyFinalized(K); +} + +void RTDyldObjectLinkingLayer2::mapSectionAddress( + VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) const { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto ActiveRTDyldItr = ActiveRTDylds.find(K); + + assert(ActiveRTDyldItr != ActiveRTDylds.end() && + "No active RTDyld instance found for key"); + ActiveRTDyldItr->second->mapSectionAddress(LocalAddress, TargetAddr); +} + +} // End namespace orc. +} // End namespace llvm. |