diff options
Diffstat (limited to 'lib/ExecutionEngine/Orc')
21 files changed, 2737 insertions, 1448 deletions
diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt index a7500ef20f37..9ca409f81cd3 100644 --- a/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -5,6 +5,8 @@ add_llvm_library(LLVMOrcJIT IndirectionUtils.cpp IRCompileLayer.cpp IRTransformLayer.cpp + JITTargetMachineBuilder.cpp + LazyReexports.cpp Legacy.cpp Layer.cpp LLJIT.cpp @@ -16,6 +18,7 @@ add_llvm_library(LLVMOrcJIT OrcMCJITReplacement.cpp RPCUtils.cpp RTDyldObjectLinkingLayer.cpp + ThreadSafeModule.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index d42e7b05ba67..241eb3600da7 100644 --- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -8,201 +8,86 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" -#include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Utils/Cloning.h" using namespace llvm; using namespace llvm::orc; -namespace { +static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM, + StringRef Suffix, + GVPredicate ShouldExtract) { -template <typename MaterializerFtor> -class LambdaValueMaterializer final : public ValueMaterializer { -public: - LambdaValueMaterializer(MaterializerFtor M) : M(std::move(M)) {} - - Value *materialize(Value *V) final { return M(V); } - -private: - MaterializerFtor M; -}; + auto DeleteExtractedDefs = [](GlobalValue &GV) { + // Bump the linkage: this global will be provided by the external module. + GV.setLinkage(GlobalValue::ExternalLinkage); -template <typename MaterializerFtor> -LambdaValueMaterializer<MaterializerFtor> -createLambdaValueMaterializer(MaterializerFtor M) { - return LambdaValueMaterializer<MaterializerFtor>(std::move(M)); -} -} // namespace - -static void extractAliases(MaterializationResponsibility &R, Module &M, - MangleAndInterner &Mangle) { - SymbolAliasMap Aliases; - - std::vector<GlobalAlias *> ModAliases; - for (auto &A : M.aliases()) - ModAliases.push_back(&A); - - for (auto *A : ModAliases) { - Constant *Aliasee = A->getAliasee(); - assert(A->hasName() && "Anonymous alias?"); - assert(Aliasee->hasName() && "Anonymous aliasee"); - std::string AliasName = A->getName(); - - Aliases[Mangle(AliasName)] = SymbolAliasMapEntry( - {Mangle(Aliasee->getName()), JITSymbolFlags::fromGlobalValue(*A)}); - - if (isa<Function>(Aliasee)) { - auto *F = cloneFunctionDecl(M, *cast<Function>(Aliasee)); - A->replaceAllUsesWith(F); - A->eraseFromParent(); - F->setName(AliasName); - } else if (isa<GlobalValue>(Aliasee)) { - auto *G = cloneGlobalVariableDecl(M, *cast<GlobalVariable>(Aliasee)); - A->replaceAllUsesWith(G); - A->eraseFromParent(); - G->setName(AliasName); - } - } - - R.replace(symbolAliases(std::move(Aliases))); -} - -static std::unique_ptr<Module> -extractAndClone(Module &M, LLVMContext &NewContext, StringRef Suffix, - function_ref<bool(const GlobalValue *)> ShouldCloneDefinition) { - SmallVector<char, 1> ClonedModuleBuffer; - - { - std::set<GlobalValue *> ClonedDefsInSrc; - ValueToValueMapTy VMap; - auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) { - if (ShouldCloneDefinition(GV)) { - ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); - return true; - } - return false; - }); - - for (auto *GV : ClonedDefsInSrc) { - // Delete the definition and bump the linkage in the source module. - if (isa<Function>(GV)) { - auto &F = *cast<Function>(GV); - F.deleteBody(); - F.setPersonalityFn(nullptr); - } else if (isa<GlobalVariable>(GV)) { - cast<GlobalVariable>(GV)->setInitializer(nullptr); + // Delete the definition in the source module. + if (isa<Function>(GV)) { + auto &F = cast<Function>(GV); + F.deleteBody(); + F.setPersonalityFn(nullptr); + } else if (isa<GlobalVariable>(GV)) { + cast<GlobalVariable>(GV).setInitializer(nullptr); + } else if (isa<GlobalAlias>(GV)) { + // We need to turn deleted aliases into function or variable decls based + // on the type of their aliasee. + auto &A = cast<GlobalAlias>(GV); + Constant *Aliasee = A.getAliasee(); + assert(A.hasName() && "Anonymous alias?"); + assert(Aliasee->hasName() && "Anonymous aliasee"); + std::string AliasName = A.getName(); + + if (isa<Function>(Aliasee)) { + auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee)); + A.replaceAllUsesWith(F); + A.eraseFromParent(); + F->setName(AliasName); + } else if (isa<GlobalVariable>(Aliasee)) { + auto *G = cloneGlobalVariableDecl(*A.getParent(), + *cast<GlobalVariable>(Aliasee)); + A.replaceAllUsesWith(G); + A.eraseFromParent(); + G->setName(AliasName); } else - llvm_unreachable("Unsupported global type"); + llvm_unreachable("Alias to unsupported type"); + } else + llvm_unreachable("Unsupported global type"); + }; - GV->setLinkage(GlobalValue::ExternalLinkage); - } - - BitcodeWriter BCWriter(ClonedModuleBuffer); - - BCWriter.writeModule(*Tmp); - BCWriter.writeSymtab(); - BCWriter.writeStrtab(); - } - - MemoryBufferRef ClonedModuleBufferRef( - StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), - "cloned module buffer"); + auto NewTSMod = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs); + auto &M = *NewTSMod.getModule(); + M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str()); - auto ClonedModule = - cantFail(parseBitcodeFile(ClonedModuleBufferRef, NewContext)); - ClonedModule->setModuleIdentifier((M.getName() + Suffix).str()); - return ClonedModule; -} - -static std::unique_ptr<Module> extractGlobals(Module &M, - LLVMContext &NewContext) { - return extractAndClone(M, NewContext, ".globals", [](const GlobalValue *GV) { - return isa<GlobalVariable>(GV); - }); + return NewTSMod; } namespace llvm { namespace orc { -class ExtractingIRMaterializationUnit : public IRMaterializationUnit { +class PartitioningIRMaterializationUnit : public IRMaterializationUnit { public: - ExtractingIRMaterializationUnit(ExecutionSession &ES, - CompileOnDemandLayer2 &Parent, - std::unique_ptr<Module> M) - : IRMaterializationUnit(ES, std::move(M)), Parent(Parent) {} - - ExtractingIRMaterializationUnit(std::unique_ptr<Module> M, - SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition, - CompileOnDemandLayer2 &Parent) - : IRMaterializationUnit(std::move(M), std::move(SymbolFlags), + PartitioningIRMaterializationUnit(ExecutionSession &ES, ThreadSafeModule TSM, + VModuleKey K, CompileOnDemandLayer &Parent) + : IRMaterializationUnit(ES, std::move(TSM), std::move(K)), + Parent(Parent) {} + + PartitioningIRMaterializationUnit( + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition, + CompileOnDemandLayer &Parent) + : IRMaterializationUnit(std::move(TSM), std::move(K), + std::move(SymbolFlags), std::move(SymbolToDefinition)), Parent(Parent) {} private: void materialize(MaterializationResponsibility R) override { - // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the - // extracted module key, extracted module, and source module key - // together. This could be used, for example, to provide a specific - // memory manager instance to the linking layer. - - auto RequestedSymbols = R.getRequestedSymbols(); - - // Extract the requested functions into a new module. - std::unique_ptr<Module> ExtractedFunctionsModule; - if (!RequestedSymbols.empty()) { - std::string Suffix; - std::set<const GlobalValue *> FunctionsToClone; - for (auto &Name : RequestedSymbols) { - auto I = SymbolToDefinition.find(Name); - assert(I != SymbolToDefinition.end() && I->second != nullptr && - "Should have a non-null definition"); - FunctionsToClone.insert(I->second); - Suffix += "."; - Suffix += *Name; - } - - std::lock_guard<std::mutex> Lock(SourceModuleMutex); - ExtractedFunctionsModule = - extractAndClone(*M, Parent.GetAvailableContext(), Suffix, - [&](const GlobalValue *GV) -> bool { - return FunctionsToClone.count(GV); - }); - } - - // Build a new ExtractingIRMaterializationUnit to delegate the unrequested - // symbols to. - SymbolFlagsMap DelegatedSymbolFlags; - IRMaterializationUnit::SymbolNameToDefinitionMap - DelegatedSymbolToDefinition; - for (auto &KV : SymbolToDefinition) { - if (RequestedSymbols.count(KV.first)) - continue; - DelegatedSymbolFlags[KV.first] = - JITSymbolFlags::fromGlobalValue(*KV.second); - DelegatedSymbolToDefinition[KV.first] = KV.second; - } - - if (!DelegatedSymbolFlags.empty()) { - assert(DelegatedSymbolFlags.size() == - DelegatedSymbolToDefinition.size() && - "SymbolFlags and SymbolToDefinition should have the same number " - "of entries"); - R.replace(llvm::make_unique<ExtractingIRMaterializationUnit>( - std::move(M), std::move(DelegatedSymbolFlags), - std::move(DelegatedSymbolToDefinition), Parent)); - } - - if (ExtractedFunctionsModule) - Parent.emitExtractedFunctionsModule(std::move(R), - std::move(ExtractedFunctionsModule)); + Parent.emitPartition(std::move(R), std::move(TSM), + std::move(SymbolToDefinition)); } - void discard(const VSO &V, SymbolStringPtr Name) override { + void discard(const JITDylib &V, const SymbolStringPtr &Name) override { // All original symbols were materialized by the CODLayer and should be // final. The function bodies provided by M should never be overridden. llvm_unreachable("Discard should never be called on an " @@ -210,44 +95,98 @@ private: } mutable std::mutex SourceModuleMutex; - CompileOnDemandLayer2 &Parent; + CompileOnDemandLayer &Parent; }; -CompileOnDemandLayer2::CompileOnDemandLayer2( - ExecutionSession &ES, IRLayer &BaseLayer, JITCompileCallbackManager &CCMgr, - IndirectStubsManagerBuilder BuildIndirectStubsManager, - GetAvailableContextFunction GetAvailableContext) - : IRLayer(ES), BaseLayer(BaseLayer), CCMgr(CCMgr), - BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)), - GetAvailableContext(std::move(GetAvailableContext)) {} - -Error CompileOnDemandLayer2::add(VSO &V, VModuleKey K, - std::unique_ptr<Module> M) { - return IRLayer::add(V, K, std::move(M)); +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileRequested(GlobalValueSet Requested) { + return std::move(Requested); } -void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K, - std::unique_ptr<Module> M) { +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) { + return None; +} + +CompileOnDemandLayer::CompileOnDemandLayer( + ExecutionSession &ES, IRLayer &BaseLayer, LazyCallThroughManager &LCTMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager) + : IRLayer(ES), BaseLayer(BaseLayer), LCTMgr(LCTMgr), + BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)) {} + +void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { + this->Partition = std::move(Partition); +} + +void CompileOnDemandLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Null module"); + auto &ES = getExecutionSession(); - assert(M && "M should not be null"); + auto &M = *TSM.getModule(); + + // First, do some cleanup on the module: + cleanUpModule(M); + + // Now sort the callables and non-callables, build re-exports and lodge the + // actual module with the implementation dylib. + auto &PDR = getPerDylibResources(R.getTargetJITDylib()); - for (auto &GV : M->global_values()) - if (GV.hasWeakLinkage()) - GV.setLinkage(GlobalValue::ExternalLinkage); + MangleAndInterner Mangle(ES, M.getDataLayout()); + SymbolAliasMap NonCallables; + SymbolAliasMap Callables; + for (auto &GV : M.global_values()) { + if (GV.isDeclaration() || GV.hasLocalLinkage() || GV.hasAppendingLinkage()) + continue; - MangleAndInterner Mangle(ES, M->getDataLayout()); + auto Name = Mangle(GV.getName()); + auto Flags = JITSymbolFlags::fromGlobalValue(GV); + if (Flags.isCallable()) + Callables[Name] = SymbolAliasMapEntry(Name, Flags); + else + NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); + } + + // Create a partitioning materialization unit and lodge it with the + // implementation dylib. + if (auto Err = PDR.getImplDylib().define( + llvm::make_unique<PartitioningIRMaterializationUnit>( + ES, std::move(TSM), R.getVModuleKey(), *this))) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } - extractAliases(R, *M, Mangle); + R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), true)); + R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), + std::move(Callables))); +} - auto GlobalsModule = extractGlobals(*M, GetAvailableContext()); +CompileOnDemandLayer::PerDylibResources & +CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { + auto I = DylibResources.find(&TargetD); + if (I == DylibResources.end()) { + auto &ImplD = getExecutionSession().createJITDylib( + TargetD.getName() + ".impl", false); + TargetD.withSearchOrderDo([&](const JITDylibSearchList &TargetSearchOrder) { + auto NewSearchOrder = TargetSearchOrder; + assert(!NewSearchOrder.empty() && + NewSearchOrder.front().first == &TargetD && + NewSearchOrder.front().second == true && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewSearchOrder.insert(std::next(NewSearchOrder.begin()), {&ImplD, true}); + ImplD.setSearchOrder(std::move(NewSearchOrder), false); + }); + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); + I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; + } - // Delete the bodies of any available externally functions, rename the - // rest, and build the compile callbacks. - std::map<SymbolStringPtr, std::pair<JITTargetAddress, JITSymbolFlags>> - StubCallbacksAndLinkages; - auto &TargetVSO = R.getTargetVSO(); + return I->second; +} - for (auto &F : M->functions()) { +void CompileOnDemandLayer::cleanUpModule(Module &M) { + for (auto &F : M.functions()) { if (F.isDeclaration()) continue; @@ -256,87 +195,108 @@ void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K, F.setPersonalityFn(nullptr); continue; } + } +} - assert(F.hasName() && "Function should have a name"); - std::string StubUnmangledName = F.getName(); - F.setName(F.getName() + "$body"); - auto StubDecl = cloneFunctionDecl(*M, F); - StubDecl->setName(StubUnmangledName); - StubDecl->setPersonalityFn(nullptr); - StubDecl->setLinkage(GlobalValue::ExternalLinkage); - F.replaceAllUsesWith(StubDecl); - - auto StubName = Mangle(StubUnmangledName); - auto BodyName = Mangle(F.getName()); - if (auto CallbackAddr = CCMgr.getCompileCallback( - [BodyName, &TargetVSO, &ES]() -> JITTargetAddress { - if (auto Sym = lookup({&TargetVSO}, BodyName)) - return Sym->getAddress(); - else { - ES.reportError(Sym.takeError()); - return 0; - } - })) { - auto Flags = JITSymbolFlags::fromGlobalValue(F); - Flags &= ~JITSymbolFlags::Weak; - StubCallbacksAndLinkages[std::move(StubName)] = - std::make_pair(*CallbackAddr, Flags); - } else { - ES.reportError(CallbackAddr.takeError()); - R.failMaterialization(); - return; - } +void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { + // Expands the partition to ensure the following rules hold: + // (1) If any alias is in the partition, its aliasee is also in the partition. + // (2) If any aliasee is in the partition, its aliases are also in the + // partiton. + // (3) If any global variable is in the partition then all global variables + // are in the partition. + assert(!Partition.empty() && "Unexpected empty partition"); + + const Module &M = *(*Partition.begin())->getParent(); + bool ContainsGlobalVariables = false; + std::vector<const GlobalValue *> GVsToAdd; + + for (auto *GV : Partition) + if (isa<GlobalAlias>(GV)) + GVsToAdd.push_back( + cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee())); + else if (isa<GlobalVariable>(GV)) + ContainsGlobalVariables = true; + + for (auto &A : M.aliases()) + if (Partition.count(cast<GlobalValue>(A.getAliasee()))) + GVsToAdd.push_back(&A); + + if (ContainsGlobalVariables) + for (auto &G : M.globals()) + GVsToAdd.push_back(&G); + + for (auto *GV : GVsToAdd) + Partition.insert(GV); +} + +void CompileOnDemandLayer::emitPartition( + MaterializationResponsibility R, ThreadSafeModule TSM, + IRMaterializationUnit::SymbolNameToDefinitionMap Defs) { + + // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the + // extracted module key, extracted module, and source module key + // together. This could be used, for example, to provide a specific + // memory manager instance to the linking layer. + + auto &ES = getExecutionSession(); + + GlobalValueSet RequestedGVs; + for (auto &Name : R.getRequestedSymbols()) { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); } - // Build the stub inits map. - IndirectStubsManager::StubInitsMap StubInits; - for (auto &KV : StubCallbacksAndLinkages) - StubInits[*KV.first] = KV.second; + auto GVsToExtract = Partition(RequestedGVs); - // Build the function-body-extracting materialization unit. - if (auto Err = R.getTargetVSO().define( - llvm::make_unique<ExtractingIRMaterializationUnit>(ES, *this, - std::move(M)))) { - ES.reportError(std::move(Err)); - R.failMaterialization(); + // Take a 'None' partition to mean the whole module (as opposed to an empty + // partition, which means "materialize nothing"). Emit the whole module + // unmodified to the base layer. + if (GVsToExtract == None) { + Defs.clear(); + BaseLayer.emit(std::move(R), std::move(TSM)); return; } - // Build the stubs. - // FIXME: Remove function bodies materialization unit if stub creation fails. - auto &StubsMgr = getStubsManager(TargetVSO); - if (auto Err = StubsMgr.createStubs(StubInits)) { - ES.reportError(std::move(Err)); - R.failMaterialization(); + // If the partition is empty, return the whole module to the symbol table. + if (GVsToExtract->empty()) { + R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + std::move(TSM), R.getSymbols(), std::move(Defs), *this)); return; } - // Resolve and finalize stubs. - SymbolMap ResolvedStubs; - for (auto &KV : StubCallbacksAndLinkages) { - if (auto Sym = StubsMgr.findStub(*KV.first, false)) - ResolvedStubs[KV.first] = Sym; - else - llvm_unreachable("Stub went missing"); + // Ok -- we actually need to partition the symbols. Promote the symbol + // linkages/names. + // FIXME: We apply this once per partitioning. It's safe, but overkill. + { + auto PromotedGlobals = PromoteSymbols(*TSM.getModule()); + if (!PromotedGlobals.empty()) { + MangleAndInterner Mangle(ES, TSM.getModule()->getDataLayout()); + SymbolFlagsMap SymbolFlags; + for (auto &GV : PromotedGlobals) + SymbolFlags[Mangle(GV->getName())] = + JITSymbolFlags::fromGlobalValue(*GV); + if (auto Err = R.defineMaterializing(SymbolFlags)) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + } } - R.resolve(ResolvedStubs); + expandPartition(*GVsToExtract); - BaseLayer.emit(std::move(R), std::move(K), std::move(GlobalsModule)); -} + // Extract the requested partiton (plus any necessary aliases) and + // put the rest back into the impl dylib. + auto ShouldExtract = [&](const GlobalValue &GV) -> bool { + return GVsToExtract->count(&GV); + }; -IndirectStubsManager &CompileOnDemandLayer2::getStubsManager(const VSO &V) { - std::lock_guard<std::mutex> Lock(CODLayerMutex); - StubManagersMap::iterator I = StubsMgrs.find(&V); - if (I == StubsMgrs.end()) - I = StubsMgrs.insert(std::make_pair(&V, BuildIndirectStubsManager())).first; - return *I->second; -} + auto ExtractedTSM = extractSubModule(TSM, ".submodule", ShouldExtract); + R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + ES, std::move(TSM), R.getVModuleKey(), *this)); -void CompileOnDemandLayer2::emitExtractedFunctionsModule( - MaterializationResponsibility R, std::unique_ptr<Module> M) { - auto K = getExecutionSession().allocateVModule(); - BaseLayer.emit(std::move(R), std::move(K), std::move(M)); + BaseLayer.emit(std::move(R), std::move(ExtractedTSM)); } } // end namespace orc diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index 4325d57f73d0..73c0bcdf7d28 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -1,4 +1,4 @@ -//===----- Core.cpp - Core ORC APIs (MaterializationUnit, VSO, etc.) ------===// +//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// // // The LLVM Compiler Infrastructure // @@ -11,6 +11,7 @@ #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/IR/Mangler.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" @@ -18,98 +19,203 @@ #include <future> #endif +#define DEBUG_TYPE "orc" + +using namespace llvm; + +namespace { + +#ifndef NDEBUG + +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(false), + cl::desc("debug print hidden symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(false), + cl::desc("debug print callable symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(false), + cl::desc("debug print data symbols defined by " + "materialization units"), + cl::Hidden); + +#endif // NDEBUG + +// SetPrinter predicate that prints every element. +template <typename T> struct PrintAll { + bool operator()(const T &E) { return true; } +}; + +bool anyPrintSymbolOptionSet() { +#ifndef NDEBUG + return PrintHidden || PrintCallable || PrintData; +#else + return false; +#endif // NDEBUG +} + +bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { +#ifndef NDEBUG + // Bail out early if this is a hidden symbol and we're not printing hiddens. + if (!PrintHidden && !Flags.isExported()) + return false; + + // Return true if this is callable and we're printing callables. + if (PrintCallable && Flags.isCallable()) + return true; + + // Return true if this is data and we're printing data. + if (PrintData && !Flags.isCallable()) + return true; + + // otherwise return false. + return false; +#else + return false; +#endif // NDEBUG +} + +// Prints a set of items, filtered by an user-supplied predicate. +template <typename Set, typename Pred = PrintAll<typename Set::value_type>> +class SetPrinter { +public: + SetPrinter(const Set &S, Pred ShouldPrint = Pred()) + : S(S), ShouldPrint(std::move(ShouldPrint)) {} + + void printTo(llvm::raw_ostream &OS) const { + bool PrintComma = false; + OS << "{"; + for (auto &E : S) { + if (ShouldPrint(E)) { + if (PrintComma) + OS << ','; + OS << ' ' << E; + PrintComma = true; + } + } + OS << " }"; + } + +private: + const Set &S; + mutable Pred ShouldPrint; +}; + +template <typename Set, typename Pred> +SetPrinter<Set, Pred> printSet(const Set &S, Pred P = Pred()) { + return SetPrinter<Set, Pred>(S, std::move(P)); +} + +// Render a SetPrinter by delegating to its printTo method. +template <typename Set, typename Pred> +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, + const SetPrinter<Set, Pred> &Printer) { + Printer.printTo(OS); + return OS; +} + +struct PrintSymbolFlagsMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolFlagsMap::value_type &KV) { + return flagsMatchCLOpts(KV.second); + } +}; + +struct PrintSymbolMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolMap::value_type &KV) { + return flagsMatchCLOpts(KV.second.getFlags()); + } +}; + +} // end anonymous namespace + namespace llvm { namespace orc { + SymbolStringPool::PoolMapEntry SymbolStringPtr::Tombstone(0); + char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; +char SymbolsCouldNotBeRemoved::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = RegisterDependenciesFunction(); void MaterializationUnit::anchor() {} +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { + return OS << *Sym; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + return OS << printSet(Symbols, PrintAll<SymbolStringPtr>()); +} + raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.isCallable()) + OS << "[Callable]"; + else + OS << "[Data]"; if (Flags.isWeak()) - OS << 'W'; + OS << "[Weak]"; else if (Flags.isCommon()) - OS << 'C'; - else - OS << 'S'; + OS << "[Common]"; - if (Flags.isExported()) - OS << 'E'; - else - OS << 'H'; + if (!Flags.isExported()) + OS << "[Hidden]"; return OS; } raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { - OS << format("0x%016x", Sym.getAddress()) << " " << Sym.getFlags(); - return OS; + return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " + << Sym.getFlags(); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { + return OS << "(\"" << KV.first << "\", " << KV.second << ")"; } raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { - OS << "\"" << *KV.first << "\": " << KV.second; - return OS; + return OS << "(\"" << KV.first << "\": " << KV.second << ")"; } -raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { - OS << "{"; - if (!Symbols.empty()) { - OS << " \"" << **Symbols.begin() << "\""; - for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) - OS << ", \"" << *Sym << "\""; - } - OS << " }"; - return OS; +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + return OS << printSet(SymbolFlags, PrintSymbolFlagsMapElemsMatchingCLOpts()); } raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { - OS << "{"; - if (!Symbols.empty()) { - OS << " {" << *Symbols.begin() << "}"; - for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) - OS << ", {" << Sym << "}"; - } - OS << " }"; - return OS; + return OS << printSet(Symbols, PrintSymbolMapElemsMatchingCLOpts()); } -raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { - OS << "{"; - if (!SymbolFlags.empty()) { - OS << " {\"" << *SymbolFlags.begin()->first - << "\": " << SymbolFlags.begin()->second << "}"; - for (auto &KV : - make_range(std::next(SymbolFlags.begin()), SymbolFlags.end())) - OS << ", {\"" << *KV.first << "\": " << KV.second << "}"; - } - OS << " }"; - return OS; +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV) { + return OS << "(" << KV.first << ", " << KV.second << ")"; } raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { - OS << "{"; - if (!Deps.empty()) { - OS << " { " << Deps.begin()->first->getName() << ": " - << Deps.begin()->second << " }"; - for (auto &KV : make_range(std::next(Deps.begin()), Deps.end())) - OS << ", { " << KV.first->getName() << ": " << KV.second << " }"; - } - OS << " }"; - return OS; + return OS << printSet(Deps, PrintAll<SymbolDependenceMap::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { + OS << "MU@" << &MU << " (\"" << MU.getName() << "\""; + if (anyPrintSymbolOptionSet()) + OS << ", " << MU.getSymbols(); + return OS << ")"; } -raw_ostream &operator<<(raw_ostream &OS, const VSOList &VSOs) { +raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { OS << "["; - if (!VSOs.empty()) { - assert(VSOs.front() && "VSOList entries must not be null"); - OS << " " << VSOs.front()->getName(); - for (auto *V : make_range(std::next(VSOs.begin()), VSOs.end())) { - assert(V && "VSOList entries must not be null"); - OS << ", " << V->getName(); + if (!JDs.empty()) { + assert(JDs.front().first && "JITDylibList entries must not be null"); + OS << " (\"" << JDs.front().first->getName() << "\", " + << (JDs.front().second ? "true" : "false") << ")"; + for (auto &KV : make_range(std::next(JDs.begin()), JDs.end())) { + assert(KV.first && "JITDylibList entries must not be null"); + OS << ", (\"" << KV.first->getName() << "\", " + << (KV.second ? "true" : "false") << ")"; } } OS << " ]"; @@ -142,359 +248,17 @@ void SymbolsNotFound::log(raw_ostream &OS) const { OS << "Symbols not found: " << Symbols; } -void ExecutionSessionBase::legacyFailQuery(AsynchronousSymbolQuery &Q, - Error Err) { - assert(!!Err && "Error should be in failure state"); - - bool SendErrorToQuery; - runSessionLocked([&]() { - Q.detach(); - SendErrorToQuery = Q.canStillFail(); - }); - - if (SendErrorToQuery) - Q.handleFailed(std::move(Err)); - else - reportError(std::move(Err)); -} - -Expected<SymbolMap> ExecutionSessionBase::legacyLookup( - ExecutionSessionBase &ES, LegacyAsyncLookupFunction AsyncLookup, - SymbolNameSet Names, bool WaitUntilReady, - RegisterDependenciesFunction RegisterDependencies) { -#if LLVM_ENABLE_THREADS - // In the threaded case we use promises to return the results. - std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; - Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { - if (R) - PromisedResult.set_value(std::move(*R)); - else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } - PromisedResult.set_value(SymbolMap()); - } - }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - ES.reportError(std::move(Err)); - }; - } - -#else - SymbolMap Result; - Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - - auto OnResolve = [&](Expected<SymbolMap> R) { - ErrorAsOutParameter _(&ResolutionError); - if (R) - Result = std::move(*R); - else - ResolutionError = R.takeError(); - }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - ES.reportError(std::move(Err)); - }; - } -#endif - - auto Query = std::make_shared<AsynchronousSymbolQuery>( - Names, std::move(OnResolve), std::move(OnReady)); - // FIXME: This should be run session locked along with the registration code - // and error reporting below. - SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); - - // If the query was lodged successfully then register the dependencies, - // otherwise fail it with an error. - if (UnresolvedSymbols.empty()) - RegisterDependencies(Query->QueryRegistrations); - else { - bool DeliverError = runSessionLocked([&]() { - Query->detach(); - return Query->canStillFail(); - }); - auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); - if (DeliverError) - Query->handleFailed(std::move(Err)); - else - ES.reportError(std::move(Err)); - } - -#if LLVM_ENABLE_THREADS - auto ResultFuture = PromisedResult.get_future(); - auto Result = ResultFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); - - return std::move(Result); - -#else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); - - return Result; -#endif -} - -void ExecutionSessionBase::lookup( - const VSOList &VSOs, const SymbolNameSet &Symbols, - SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, - RegisterDependenciesFunction RegisterDependencies) { - - // lookup can be re-entered recursively if running on a single thread. Run any - // outstanding MUs in case this query depends on them, otherwise the main - // thread will starve waiting for a result from an MU that it failed to run. - runOutstandingMUs(); - - auto Unresolved = std::move(Symbols); - std::map<VSO *, MaterializationUnitList> MUsMap; - auto Q = std::make_shared<AsynchronousSymbolQuery>( - Symbols, std::move(OnResolve), std::move(OnReady)); - bool QueryIsFullyResolved = false; - bool QueryIsFullyReady = false; - bool QueryFailed = false; - - runSessionLocked([&]() { - for (auto *V : VSOs) { - assert(V && "VSOList entries must not be null"); - assert(!MUsMap.count(V) && - "VSOList should not contain duplicate entries"); - V->lodgeQuery(Q, Unresolved, MUsMap[V]); - } - - if (Unresolved.empty()) { - // Query lodged successfully. - - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryIsFullyResolved = Q->isFullyResolved(); - QueryIsFullyReady = Q->isFullyReady(); - - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - } else { - // Query failed due to unresolved symbols. - QueryFailed = true; - - // Disconnect the query from its dependencies. - Q->detach(); - - // Replace the MUs. - for (auto &KV : MUsMap) - for (auto &MU : KV.second) - KV.first->replace(std::move(MU)); - } - }); - - if (QueryFailed) { - Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); - return; - } else { - if (QueryIsFullyResolved) - Q->handleFullyResolved(); - if (QueryIsFullyReady) - Q->handleFullyReady(); - } - - // Move the MUs to the OutstandingMUs list, then materialize. - { - std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); - - for (auto &KV : MUsMap) - for (auto &MU : KV.second) - OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); - } - - runOutstandingMUs(); +SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); } -Expected<SymbolMap> -ExecutionSessionBase::lookup(const VSOList &VSOs, const SymbolNameSet &Symbols, - RegisterDependenciesFunction RegisterDependencies, - bool WaitUntilReady) { -#if LLVM_ENABLE_THREADS - // In the threaded case we use promises to return the results. - std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; - Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { - if (R) - PromisedResult.set_value(std::move(*R)); - else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } - PromisedResult.set_value(SymbolMap()); - } - }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - -#else - SymbolMap Result; - Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - - auto OnResolve = [&](Expected<SymbolMap> R) { - ErrorAsOutParameter _(&ResolutionError); - if (R) - Result = std::move(*R); - else - ResolutionError = R.takeError(); - }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } -#endif - - // Perform the asynchronous lookup. - lookup(VSOs, Symbols, OnResolve, OnReady, RegisterDependencies); - -#if LLVM_ENABLE_THREADS - auto ResultFuture = PromisedResult.get_future(); - auto Result = ResultFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); - - return std::move(Result); - -#else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); - - return Result; -#endif +std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); } -void ExecutionSessionBase::runOutstandingMUs() { - while (1) { - std::pair<VSO *, std::unique_ptr<MaterializationUnit>> VSOAndMU; - - { - std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); - if (!OutstandingMUs.empty()) { - VSOAndMU = std::move(OutstandingMUs.back()); - OutstandingMUs.pop_back(); - } - } - - if (VSOAndMU.first) { - assert(VSOAndMU.second && "VSO, but no MU?"); - dispatchMaterialization(*VSOAndMU.first, std::move(VSOAndMU.second)); - } else - break; - } +void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { + OS << "Symbols could not be removed: " << Symbols; } AsynchronousSymbolQuery::AsynchronousSymbolQuery( @@ -520,23 +284,45 @@ void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, void AsynchronousSymbolQuery::handleFullyResolved() { assert(NotYetResolvedCount == 0 && "Not fully resolved?"); - assert(NotifySymbolsResolved && - "NotifySymbolsResolved already called or error occurred"); - NotifySymbolsResolved(std::move(ResolvedSymbols)); + + if (!NotifySymbolsResolved) { + // handleFullyResolved may be called by handleFullyReady (see comments in + // that method), in which case this is a no-op, so bail out. + assert(!NotifySymbolsReady && + "NotifySymbolsResolved already called or an error occurred"); + return; + } + + auto TmpNotifySymbolsResolved = std::move(NotifySymbolsResolved); NotifySymbolsResolved = SymbolsResolvedCallback(); + TmpNotifySymbolsResolved(std::move(ResolvedSymbols)); } void AsynchronousSymbolQuery::notifySymbolReady() { - assert(NotYetReadyCount != 0 && "All symbols already finalized"); + assert(NotYetReadyCount != 0 && "All symbols already emitted"); --NotYetReadyCount; } void AsynchronousSymbolQuery::handleFullyReady() { + assert(NotifySymbolsReady && + "NotifySymbolsReady already called or an error occurred"); + + auto TmpNotifySymbolsReady = std::move(NotifySymbolsReady); + NotifySymbolsReady = SymbolsReadyCallback(); + + if (NotYetResolvedCount == 0 && NotifySymbolsResolved) { + // The NotifyResolved callback of one query must have caused this query to + // become ready (i.e. there is still a handleFullyResolved callback waiting + // to be made back up the stack). Fold the handleFullyResolved call into + // this one before proceeding. This will cause the call further up the + // stack to become a no-op. + handleFullyResolved(); + } + assert(QueryRegistrations.empty() && "Query is still registered with some symbols"); assert(!NotifySymbolsResolved && "Resolution not applied yet"); - NotifySymbolsReady(Error::success()); - NotifySymbolsReady = SymbolsReadyCallback(); + TmpNotifySymbolsReady(Error::success()); } bool AsynchronousSymbolQuery::canStillFail() { @@ -557,17 +343,19 @@ void AsynchronousSymbolQuery::handleFailed(Error Err) { NotifySymbolsReady = SymbolsReadyCallback(); } -void AsynchronousSymbolQuery::addQueryDependence(VSO &V, SymbolStringPtr Name) { - bool Added = QueryRegistrations[&V].insert(std::move(Name)).second; +void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, + SymbolStringPtr Name) { + bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second; (void)Added; assert(Added && "Duplicate dependence notification?"); } void AsynchronousSymbolQuery::removeQueryDependence( - VSO &V, const SymbolStringPtr &Name) { - auto QRI = QueryRegistrations.find(&V); - assert(QRI != QueryRegistrations.end() && "No dependencies registered for V"); - assert(QRI->second.count(Name) && "No dependency on Name in V"); + JITDylib &JD, const SymbolStringPtr &Name) { + auto QRI = QueryRegistrations.find(&JD); + assert(QRI != QueryRegistrations.end() && + "No dependencies registered for JD"); + assert(QRI->second.count(Name) && "No dependency on Name in JD"); QRI->second.erase(Name); if (QRI->second.empty()) QueryRegistrations.erase(QRI); @@ -583,8 +371,8 @@ void AsynchronousSymbolQuery::detach() { } MaterializationResponsibility::MaterializationResponsibility( - VSO &V, SymbolFlagsMap SymbolFlags) - : V(V), SymbolFlags(std::move(SymbolFlags)) { + JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) + : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { assert(!this->SymbolFlags.empty() && "Materializing nothing?"); #ifndef NDEBUG @@ -598,11 +386,13 @@ MaterializationResponsibility::~MaterializationResponsibility() { "All symbols should have been explicitly materialized or failed"); } -SymbolNameSet MaterializationResponsibility::getRequestedSymbols() { - return V.getRequestedSymbols(SymbolFlags); +SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { + return JD.getRequestedSymbols(SymbolFlags); } void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { + LLVM_DEBUG(dbgs() << "In " << JD.getName() << " resolving " << Symbols + << "\n"); #ifndef NDEBUG for (auto &KV : Symbols) { auto I = SymbolFlags.find(KV.first); @@ -619,17 +409,17 @@ void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { } #endif - V.resolve(Symbols); + JD.resolve(Symbols); } -void MaterializationResponsibility::finalize() { +void MaterializationResponsibility::emit() { #ifndef NDEBUG for (auto &KV : SymbolFlags) assert(!KV.second.isMaterializing() && - "Failed to resolve symbol before finalization"); + "Failed to resolve symbol before emission"); #endif // NDEBUG - V.finalize(SymbolFlags); + JD.emit(SymbolFlags); SymbolFlags.clear(); } @@ -637,8 +427,8 @@ Error MaterializationResponsibility::defineMaterializing( const SymbolFlagsMap &NewSymbolFlags) { // Add the given symbols to this responsibility object. // It's ok if we hit a duplicate here: In that case the new version will be - // discarded, and the VSO::defineMaterializing method will return a duplicate - // symbol error. + // discarded, and the JITDylib::defineMaterializing method will return a + // duplicate symbol error. for (auto &KV : NewSymbolFlags) { auto I = SymbolFlags.insert(KV).first; (void)I; @@ -647,7 +437,7 @@ Error MaterializationResponsibility::defineMaterializing( #endif } - return V.defineMaterializing(NewSymbolFlags); + return JD.defineMaterializing(NewSymbolFlags); } void MaterializationResponsibility::failMaterialization() { @@ -656,7 +446,7 @@ void MaterializationResponsibility::failMaterialization() { for (auto &KV : SymbolFlags) FailedSymbols.insert(KV.first); - V.notifyFailed(FailedSymbols); + JD.notifyFailed(FailedSymbols); SymbolFlags.clear(); } @@ -665,11 +455,21 @@ void MaterializationResponsibility::replace( for (auto &KV : MU->getSymbols()) SymbolFlags.erase(KV.first); - V.replace(std::move(MU)); + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD.getName() << " replacing symbols with " << *MU + << "\n"; + });); + + JD.replace(std::move(MU)); } MaterializationResponsibility -MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) { +MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, + VModuleKey NewKey) { + + if (NewKey == VModuleKey()) + NewKey = K; + SymbolFlagsMap DelegatedFlags; for (auto &Name : Symbols) { @@ -682,34 +482,40 @@ MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) { SymbolFlags.erase(I); } - return MaterializationResponsibility(V, std::move(DelegatedFlags)); + return MaterializationResponsibility(JD, std::move(DelegatedFlags), + std::move(NewKey)); } void MaterializationResponsibility::addDependencies( const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { assert(SymbolFlags.count(Name) && "Symbol not covered by this MaterializationResponsibility instance"); - V.addDependencies(Name, Dependencies); + JD.addDependencies(Name, Dependencies); } void MaterializationResponsibility::addDependenciesForAll( const SymbolDependenceMap &Dependencies) { for (auto &KV : SymbolFlags) - V.addDependencies(KV.first, Dependencies); + JD.addDependencies(KV.first, Dependencies); } AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( - SymbolMap Symbols) - : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {} + SymbolMap Symbols, VModuleKey K) + : MaterializationUnit(extractFlags(Symbols), std::move(K)), + Symbols(std::move(Symbols)) {} + +StringRef AbsoluteSymbolsMaterializationUnit::getName() const { + return "<Absolute Symbols>"; +} void AbsoluteSymbolsMaterializationUnit::materialize( MaterializationResponsibility R) { R.resolve(Symbols); - R.finalize(); + R.emit(); } -void AbsoluteSymbolsMaterializationUnit::discard(const VSO &V, - SymbolStringPtr Name) { +void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { assert(Symbols.count(Name) && "Symbol is not part of this MU"); Symbols.erase(Name); } @@ -723,19 +529,26 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { } ReExportsMaterializationUnit::ReExportsMaterializationUnit( - VSO *SourceVSO, SymbolAliasMap Aliases) - : MaterializationUnit(extractFlags(Aliases)), SourceVSO(SourceVSO), + JITDylib *SourceJD, bool MatchNonExported, SymbolAliasMap Aliases, + VModuleKey K) + : MaterializationUnit(extractFlags(Aliases), std::move(K)), + SourceJD(SourceJD), MatchNonExported(MatchNonExported), Aliases(std::move(Aliases)) {} +StringRef ReExportsMaterializationUnit::getName() const { + return "<Reexports>"; +} + void ReExportsMaterializationUnit::materialize( MaterializationResponsibility R) { - auto &ES = R.getTargetVSO().getExecutionSession(); - VSO &TgtV = R.getTargetVSO(); - VSO &SrcV = SourceVSO ? *SourceVSO : TgtV; + auto &ES = R.getTargetJITDylib().getExecutionSession(); + JITDylib &TgtJD = R.getTargetJITDylib(); + JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD; // Find the set of requested aliases and aliasees. Return any unrequested - // aliases back to the VSO so as to not prematurely materialize any aliasees. + // aliases back to the JITDylib so as to not prematurely materialize any + // aliasees. auto RequestedSymbols = R.getRequestedSymbols(); SymbolAliasMap RequestedAliases; @@ -747,8 +560,8 @@ void ReExportsMaterializationUnit::materialize( } if (!Aliases.empty()) { - if (SourceVSO) - R.replace(reexports(*SourceVSO, std::move(Aliases))); + if (SourceJD) + R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); else R.replace(symbolAliases(std::move(Aliases))); } @@ -776,20 +589,22 @@ void ReExportsMaterializationUnit::materialize( SymbolNameSet QuerySymbols; SymbolAliasMap QueryAliases; - for (auto I = RequestedAliases.begin(), E = RequestedAliases.end(); - I != E;) { - auto Tmp = I++; - + // Collect as many aliases as we can without including a chain. + for (auto &KV : RequestedAliases) { // Chain detected. Skip this symbol for this round. - if (&SrcV == &TgtV && (QueryAliases.count(Tmp->second.Aliasee) || - RequestedAliases.count(Tmp->second.Aliasee))) + if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) || + RequestedAliases.count(KV.second.Aliasee))) continue; - ResponsibilitySymbols.insert(Tmp->first); - QuerySymbols.insert(Tmp->second.Aliasee); - QueryAliases[Tmp->first] = std::move(Tmp->second); - RequestedAliases.erase(Tmp); + ResponsibilitySymbols.insert(KV.first); + QuerySymbols.insert(KV.second.Aliasee); + QueryAliases[KV.first] = std::move(KV.second); } + + // Remove the aliases collected this round from the RequestedAliases map. + for (auto &KV : QueryAliases) + RequestedAliases.erase(KV.first); + assert(!QuerySymbols.empty() && "Alias cycle detected!"); auto QueryInfo = std::make_shared<OnResolveInfo>( @@ -806,21 +621,21 @@ void ReExportsMaterializationUnit::materialize( QueryInfos.pop_back(); auto RegisterDependencies = [QueryInfo, - &SrcV](const SymbolDependenceMap &Deps) { + &SrcJD](const SymbolDependenceMap &Deps) { // If there were no materializing symbols, just bail out. if (Deps.empty()) return; - // Otherwise the only deps should be on SrcV. - assert(Deps.size() == 1 && Deps.count(&SrcV) && + // Otherwise the only deps should be on SrcJD. + assert(Deps.size() == 1 && Deps.count(&SrcJD) && "Unexpected dependencies for reexports"); - auto &SrcVDeps = Deps.find(&SrcV)->second; + auto &SrcJDDeps = Deps.find(&SrcJD)->second; SymbolDependenceMap PerAliasDepsMap; - auto &PerAliasDeps = PerAliasDepsMap[&SrcV]; + auto &PerAliasDeps = PerAliasDepsMap[&SrcJD]; for (auto &KV : QueryInfo->Aliases) - if (SrcVDeps.count(KV.second.Aliasee)) { + if (SrcJDDeps.count(KV.second.Aliasee)) { PerAliasDeps = {KV.second.Aliasee}; QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); } @@ -836,9 +651,9 @@ void ReExportsMaterializationUnit::materialize( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } QueryInfo->R.resolve(ResolutionMap); - QueryInfo->R.finalize(); + QueryInfo->R.emit(); } else { - auto &ES = QueryInfo->R.getTargetVSO().getExecutionSession(); + auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); ES.reportError(Result.takeError()); QueryInfo->R.failMaterialization(); } @@ -846,12 +661,14 @@ void ReExportsMaterializationUnit::materialize( auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - ES.lookup({&SrcV}, QuerySymbols, std::move(OnResolve), std::move(OnReady), + ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, + std::move(OnResolve), std::move(OnReady), std::move(RegisterDependencies)); } } -void ReExportsMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { +void ReExportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { assert(Aliases.count(Name) && "Symbol not covered by this MaterializationUnit"); Aliases.erase(Name); @@ -867,8 +684,8 @@ ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { } Expected<SymbolAliasMap> -buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols) { - auto Flags = SourceV.lookupFlags(Symbols); +buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { + auto Flags = SourceJD.lookupFlags(Symbols); if (Flags.size() != Symbols.size()) { SymbolNameSet Unresolved = Symbols; @@ -886,7 +703,33 @@ buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols) { return Result; } -Error VSO::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { +ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, + bool MatchNonExported, + SymbolPredicate Allow) + : SourceJD(SourceJD), MatchNonExported(MatchNonExported), + Allow(std::move(Allow)) {} + +SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, + const SymbolNameSet &Names) { + orc::SymbolNameSet Added; + orc::SymbolAliasMap AliasMap; + + auto Flags = SourceJD.lookupFlags(Names); + + for (auto &KV : Flags) { + if (Allow && !Allow(KV.first)) + continue; + AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); + Added.insert(KV.first); + } + + if (!Added.empty()) + cantFail(JD.define(reexports(SourceJD, AliasMap, MatchNonExported))); + + return Added; +} + +Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { return ES.runSessionLocked([&]() -> Error { std::vector<SymbolMap::iterator> AddedSyms; @@ -916,7 +759,7 @@ Error VSO::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { }); } -void VSO::replace(std::unique_ptr<MaterializationUnit> MU) { +void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); auto MustRunMU = @@ -967,13 +810,14 @@ void VSO::replace(std::unique_ptr<MaterializationUnit> MU) { ES.dispatchMaterialization(*this, std::move(MustRunMU)); } -SymbolNameSet VSO::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) { +SymbolNameSet +JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { return ES.runSessionLocked([&]() { SymbolNameSet RequestedSymbols; for (auto &KV : SymbolFlags) { - assert(Symbols.count(KV.first) && "VSO does not cover this symbol?"); - assert(Symbols[KV.first].getFlags().isMaterializing() && + assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); + assert(Symbols.find(KV.first)->second.getFlags().isMaterializing() && "getRequestedSymbols can only be called for materializing " "symbols"); auto I = MaterializingInfos.find(KV.first); @@ -988,47 +832,47 @@ SymbolNameSet VSO::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) { }); } -void VSO::addDependencies(const SymbolStringPtr &Name, - const SymbolDependenceMap &Dependencies) { +void JITDylib::addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { assert(Symbols.count(Name) && "Name not in symbol table"); assert((Symbols[Name].getFlags().isLazy() || Symbols[Name].getFlags().isMaterializing()) && "Symbol is not lazy or materializing"); auto &MI = MaterializingInfos[Name]; - assert(!MI.IsFinalized && "Can not add dependencies to finalized symbol"); + assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); for (auto &KV : Dependencies) { - assert(KV.first && "Null VSO in dependency?"); - auto &OtherVSO = *KV.first; - auto &DepsOnOtherVSO = MI.UnfinalizedDependencies[&OtherVSO]; + assert(KV.first && "Null JITDylib in dependency?"); + auto &OtherJITDylib = *KV.first; + auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; for (auto &OtherSymbol : KV.second) { #ifndef NDEBUG - // Assert that this symbol exists and has not been finalized already. - auto SymI = OtherVSO.Symbols.find(OtherSymbol); - assert(SymI != OtherVSO.Symbols.end() && + // Assert that this symbol exists and has not been emitted already. + auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); + assert(SymI != OtherJITDylib.Symbols.end() && (SymI->second.getFlags().isLazy() || SymI->second.getFlags().isMaterializing()) && - "Dependency on finalized symbol"); + "Dependency on emitted symbol"); #endif - auto &OtherMI = OtherVSO.MaterializingInfos[OtherSymbol]; + auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; - if (OtherMI.IsFinalized) - transferFinalizedNodeDependencies(MI, Name, OtherMI); - else if (&OtherVSO != this || OtherSymbol != Name) { + if (OtherMI.IsEmitted) + transferEmittedNodeDependencies(MI, Name, OtherMI); + else if (&OtherJITDylib != this || OtherSymbol != Name) { OtherMI.Dependants[this].insert(Name); - DepsOnOtherVSO.insert(OtherSymbol); + DepsOnOtherJITDylib.insert(OtherSymbol); } } - if (DepsOnOtherVSO.empty()) - MI.UnfinalizedDependencies.erase(&OtherVSO); + if (DepsOnOtherJITDylib.empty()) + MI.UnemittedDependencies.erase(&OtherJITDylib); } } -void VSO::resolve(const SymbolMap &Resolved) { +void JITDylib::resolve(const SymbolMap &Resolved) { auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { AsynchronousSymbolQuerySet FullyResolvedQueries; for (const auto &KV : Resolved) { @@ -1074,11 +918,11 @@ void VSO::resolve(const SymbolMap &Resolved) { } } -void VSO::finalize(const SymbolFlagsMap &Finalized) { +void JITDylib::emit(const SymbolFlagsMap &Emitted) { auto FullyReadyQueries = ES.runSessionLocked([&, this]() { AsynchronousSymbolQuerySet ReadyQueries; - for (const auto &KV : Finalized) { + for (const auto &KV : Emitted) { const auto &Name = KV.first; auto MII = MaterializingInfos.find(Name); @@ -1087,59 +931,59 @@ void VSO::finalize(const SymbolFlagsMap &Finalized) { auto &MI = MII->second; - // For each dependant, transfer this node's unfinalized dependencies to - // it. If the dependant node is fully finalized then notify any pending - // queries. + // For each dependant, transfer this node's emitted dependencies to + // it. If the dependant node is ready (i.e. has no unemitted + // dependencies) then notify any pending queries. for (auto &KV : MI.Dependants) { - auto &DependantVSO = *KV.first; + auto &DependantJD = *KV.first; for (auto &DependantName : KV.second) { auto DependantMII = - DependantVSO.MaterializingInfos.find(DependantName); - assert(DependantMII != DependantVSO.MaterializingInfos.end() && + DependantJD.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantJD.MaterializingInfos.end() && "Dependant should have MaterializingInfo"); auto &DependantMI = DependantMII->second; // Remove the dependant's dependency on this node. - assert(DependantMI.UnfinalizedDependencies[this].count(Name) && + assert(DependantMI.UnemittedDependencies[this].count(Name) && "Dependant does not count this symbol as a dependency?"); - DependantMI.UnfinalizedDependencies[this].erase(Name); - if (DependantMI.UnfinalizedDependencies[this].empty()) - DependantMI.UnfinalizedDependencies.erase(this); - - // Transfer unfinalized dependencies from this node to the dependant. - DependantVSO.transferFinalizedNodeDependencies(DependantMI, - DependantName, MI); - - // If the dependant is finalized and this node was the last of its - // unfinalized dependencies then notify any pending queries on the - // dependant node. - if (DependantMI.IsFinalized && - DependantMI.UnfinalizedDependencies.empty()) { + DependantMI.UnemittedDependencies[this].erase(Name); + if (DependantMI.UnemittedDependencies[this].empty()) + DependantMI.UnemittedDependencies.erase(this); + + // Transfer unemitted dependencies from this node to the dependant. + DependantJD.transferEmittedNodeDependencies(DependantMI, + DependantName, MI); + + // If the dependant is emitted and this node was the last of its + // unemitted dependencies then the dependant node is now ready, so + // notify any pending queries on the dependant node. + if (DependantMI.IsEmitted && + DependantMI.UnemittedDependencies.empty()) { assert(DependantMI.Dependants.empty() && "Dependants should be empty by now"); for (auto &Q : DependantMI.PendingQueries) { Q->notifySymbolReady(); if (Q->isFullyReady()) ReadyQueries.insert(Q); - Q->removeQueryDependence(DependantVSO, DependantName); + Q->removeQueryDependence(DependantJD, DependantName); } - // If this dependant node was fully finalized we can erase its - // MaterializingInfo and update its materializing state. - assert(DependantVSO.Symbols.count(DependantName) && + // Since this dependant is now ready, we erase its MaterializingInfo + // and update its materializing state. + assert(DependantJD.Symbols.count(DependantName) && "Dependant has no entry in the Symbols table"); - auto &DependantSym = DependantVSO.Symbols[DependantName]; - DependantSym.setFlags(static_cast<JITSymbolFlags::FlagNames>( - DependantSym.getFlags() & ~JITSymbolFlags::Materializing)); - DependantVSO.MaterializingInfos.erase(DependantMII); + auto &DependantSym = DependantJD.Symbols[DependantName]; + DependantSym.setFlags(DependantSym.getFlags() & + ~JITSymbolFlags::Materializing); + DependantJD.MaterializingInfos.erase(DependantMII); } } } MI.Dependants.clear(); - MI.IsFinalized = true; + MI.IsEmitted = true; - if (MI.UnfinalizedDependencies.empty()) { + if (MI.UnemittedDependencies.empty()) { for (auto &Q : MI.PendingQueries) { Q->notifySymbolReady(); if (Q->isFullyReady()) @@ -1149,8 +993,7 @@ void VSO::finalize(const SymbolFlagsMap &Finalized) { assert(Symbols.count(Name) && "Symbol has no entry in the Symbols table"); auto &Sym = Symbols[Name]; - Sym.setFlags(static_cast<JITSymbolFlags::FlagNames>( - Sym.getFlags() & ~JITSymbolFlags::Materializing)); + Sym.setFlags(Sym.getFlags() & ~JITSymbolFlags::Materializing); MaterializingInfos.erase(MII); } } @@ -1164,7 +1007,7 @@ void VSO::finalize(const SymbolFlagsMap &Finalized) { } } -void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) { +void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { // FIXME: This should fail any transitively dependant symbols too. @@ -1173,7 +1016,7 @@ void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) { for (auto &Name : FailedSymbols) { auto I = Symbols.find(Name); - assert(I != Symbols.end() && "Symbol not present in this VSO"); + assert(I != Symbols.end() && "Symbol not present in this JITDylib"); Symbols.erase(I); auto MII = MaterializingInfos.find(Name); @@ -1206,42 +1049,108 @@ void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) { Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); } -void VSO::setSearchOrder(VSOList NewSearchOrder, bool SearchThisVSOFirst) { - if (SearchThisVSOFirst && NewSearchOrder.front() != this) - NewSearchOrder.insert(NewSearchOrder.begin(), this); +void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, + bool SearchThisJITDylibFirst, + bool MatchNonExportedInThisDylib) { + if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } -void VSO::addToSearchOrder(VSO &V) { - ES.runSessionLocked([&]() { SearchOrder.push_back(&V); }); +void JITDylib::addToSearchOrder(JITDylib &JD, bool MatchNonExported) { + ES.runSessionLocked([&]() { + SearchOrder.push_back({&JD, MatchNonExported}); + }); } -void VSO::replaceInSearchOrder(VSO &OldV, VSO &NewV) { +void JITDylib::replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, + bool MatchNonExported) { ES.runSessionLocked([&]() { - auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &OldV); + auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), + [&](const JITDylibSearchList::value_type &KV) { + return KV.first == &OldJD; + }); if (I != SearchOrder.end()) - *I = &NewV; + *I = {&NewJD, MatchNonExported}; }); } -void VSO::removeFromSearchOrder(VSO &V) { +void JITDylib::removeFromSearchOrder(JITDylib &JD) { ES.runSessionLocked([&]() { - auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &V); + auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), + [&](const JITDylibSearchList::value_type &KV) { + return KV.first == &JD; + }); if (I != SearchOrder.end()) SearchOrder.erase(I); }); } -SymbolFlagsMap VSO::lookupFlags(const SymbolNameSet &Names) { +Error JITDylib::remove(const SymbolNameSet &Names) { + return ES.runSessionLocked([&]() -> Error { + using SymbolMaterializerItrPair = + std::pair<SymbolMap::iterator, UnmaterializedInfosMap::iterator>; + std::vector<SymbolMaterializerItrPair> SymbolsToRemove; + SymbolNameSet Missing; + SymbolNameSet Materializing; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + + // Note symbol missing. + if (I == Symbols.end()) { + Missing.insert(Name); + continue; + } + + // Note symbol materializing. + if (I->second.getFlags().isMaterializing()) { + Materializing.insert(Name); + continue; + } + + auto UMII = I->second.getFlags().isLazy() ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); + SymbolsToRemove.push_back(std::make_pair(I, UMII)); + } + + // If any of the symbols are not defined, return an error. + if (!Missing.empty()) + return make_error<SymbolsNotFound>(std::move(Missing)); + + // If any of the symbols are currently materializing, return an error. + if (!Materializing.empty()) + return make_error<SymbolsCouldNotBeRemoved>(std::move(Materializing)); + + // Remove the symbols. + for (auto &SymbolMaterializerItrPair : SymbolsToRemove) { + auto UMII = SymbolMaterializerItrPair.second; + + // If there is a materializer attached, call discard. + if (UMII != UnmaterializedInfos.end()) { + UMII->second->MU->doDiscard(*this, UMII->first); + UnmaterializedInfos.erase(UMII); + } + + auto SymI = SymbolMaterializerItrPair.first; + Symbols.erase(SymI); + } + + return Error::success(); + }); +} + +SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { return ES.runSessionLocked([&, this]() { SymbolFlagsMap Result; auto Unresolved = lookupFlagsImpl(Result, Names); - if (FallbackDefinitionGenerator && !Unresolved.empty()) { - auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); - if (!FallbackDefs.empty()) { - auto Unresolved2 = lookupFlagsImpl(Result, FallbackDefs); + if (DefGenerator && !Unresolved.empty()) { + auto NewDefs = DefGenerator(*this, Unresolved); + if (!NewDefs.empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, NewDefs); (void)Unresolved2; assert(Unresolved2.empty() && "All fallback defs should have been found by lookupFlagsImpl"); @@ -1251,8 +1160,8 @@ SymbolFlagsMap VSO::lookupFlags(const SymbolNameSet &Names) { }); } -SymbolNameSet VSO::lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names) { +SymbolNameSet JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { SymbolNameSet Unresolved; for (auto &Name : Names) { @@ -1270,38 +1179,43 @@ SymbolNameSet VSO::lookupFlagsImpl(SymbolFlagsMap &Flags, return Unresolved; } -void VSO::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, MaterializationUnitList &MUs) { +void JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs) { assert(Q && "Query can not be null"); - lodgeQueryImpl(Q, Unresolved, MUs); - if (FallbackDefinitionGenerator && !Unresolved.empty()) { - auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); - if (!FallbackDefs.empty()) { - for (auto &D : FallbackDefs) + lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); + if (DefGenerator && !Unresolved.empty()) { + auto NewDefs = DefGenerator(*this, Unresolved); + if (!NewDefs.empty()) { + for (auto &D : NewDefs) Unresolved.erase(D); - lodgeQueryImpl(Q, FallbackDefs, MUs); - assert(FallbackDefs.empty() && + lodgeQueryImpl(Q, NewDefs, MatchNonExported, MUs); + assert(NewDefs.empty() && "All fallback defs should have been found by lookupImpl"); } } } -void VSO::lodgeQueryImpl( +void JITDylib::lodgeQueryImpl( std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, + bool MatchNonExported, std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { - for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { - auto TmpI = I++; - auto Name = *TmpI; + std::vector<SymbolStringPtr> ToRemove; + for (auto Name : Unresolved) { // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); if (SymI == Symbols.end()) continue; - // If we found Name in V, remove it frome the Unresolved set and add it - // to the added set. - Unresolved.erase(TmpI); + // If this is a non exported symbol and we're skipping those then skip it. + if (!SymI->second.getFlags().isExported() && !MatchNonExported) + continue; + + // If we matched against Name in JD, mark it to be removed from the Unresolved + // set. + ToRemove.push_back(Name); // If the symbol has an address then resolve it. if (SymI->second.getAddress() != 0) @@ -1333,8 +1247,8 @@ void VSO::lodgeQueryImpl( // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing. Finalize it and - // continue. + // The symbol is neither lazy nor materializing, so it must be + // ready. Notify the query and continue. Q->notifySymbolReady(); continue; } @@ -1346,10 +1260,14 @@ void VSO::lodgeQueryImpl( MI.PendingQueries.push_back(Q); Q->addQueryDependence(*this, Name); } + + // Remove any symbols that we found. + for (auto &Name : ToRemove) + Unresolved.erase(Name); } -SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Names) { +SymbolNameSet JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { assert(Q && "Query can not be null"); ES.runOutstandingMUs(); @@ -1360,15 +1278,15 @@ SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, SymbolNameSet Unresolved = std::move(Names); ES.runSessionLocked([&, this]() { ActionFlags = lookupImpl(Q, MUs, Unresolved); - if (FallbackDefinitionGenerator && !Unresolved.empty()) { + if (DefGenerator && !Unresolved.empty()) { assert(ActionFlags == None && "ActionFlags set but unresolved symbols remain?"); - auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); - if (!FallbackDefs.empty()) { - for (auto &D : FallbackDefs) + auto NewDefs = DefGenerator(*this, Unresolved); + if (!NewDefs.empty()) { + for (auto &D : NewDefs) Unresolved.erase(D); - ActionFlags = lookupImpl(Q, MUs, FallbackDefs); - assert(FallbackDefs.empty() && + ActionFlags = lookupImpl(Q, MUs, NewDefs); + assert(NewDefs.empty() && "All fallback defs should have been found by lookupImpl"); } } @@ -1400,24 +1318,22 @@ SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, return Unresolved; } -VSO::LookupImplActionFlags -VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolNameSet &Unresolved) { +JITDylib::LookupImplActionFlags +JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { LookupImplActionFlags ActionFlags = None; + std::vector<SymbolStringPtr> ToRemove; - for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { - auto TmpI = I++; - auto Name = *TmpI; + for (auto Name : Unresolved) { // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); if (SymI == Symbols.end()) continue; - // If we found Name in V, remove it frome the Unresolved set and add it - // to the dependencies set. - Unresolved.erase(TmpI); + // If we found Name, mark it to be removed from the Unresolved set. + ToRemove.push_back(Name); // If the symbol has an address then resolve it. if (SymI->second.getAddress() != 0) { @@ -1452,8 +1368,8 @@ VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing. Finalize it and - // continue. + // The symbol is neither lazy nor materializing, so it must be ready. + // Notify the query and continue. Q->notifySymbolReady(); if (Q->isFullyReady()) ActionFlags |= NotifyFullyReady; @@ -1468,19 +1384,30 @@ VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, Q->addQueryDependence(*this, Name); } + // Remove any marked symbols from the Unresolved set. + for (auto &Name : ToRemove) + Unresolved.erase(Name); + return ActionFlags; } -void VSO::dump(raw_ostream &OS) { +void JITDylib::dump(raw_ostream &OS) { ES.runSessionLocked([&, this]() { - OS << "VSO \"" << VSOName - << "\" (ES: " << format("0x%016x", reinterpret_cast<uintptr_t>(&ES)) - << "):\n" + OS << "JITDylib \"" << JITDylibName << "\" (ES: " + << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n" + << "Search order: ["; + for (auto &KV : SearchOrder) + OS << " (\"" << KV.first->getName() << "\", " + << (KV.second ? "all" : "exported only") << ")"; + OS << " ]\n" << "Symbol table:\n"; for (auto &KV : Symbols) { - OS << " \"" << *KV.first - << "\": " << format("0x%016x", KV.second.getAddress()); + OS << " \"" << *KV.first << "\": "; + if (auto Addr = KV.second.getAddress()) + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags(); + else + OS << "<not resolved>"; if (KV.second.getFlags().isLazy() || KV.second.getFlags().isMaterializing()) { OS << " ("; @@ -1492,7 +1419,7 @@ void VSO::dump(raw_ostream &OS) { } if (KV.second.getFlags().isMaterializing()) OS << " Materializing"; - OS << " )\n"; + OS << ", " << KV.second.getFlags() << " )\n"; } else OS << "\n"; } @@ -1501,7 +1428,7 @@ void VSO::dump(raw_ostream &OS) { OS << " MaterializingInfos entries:\n"; for (auto &KV : MaterializingInfos) { OS << " \"" << *KV.first << "\":\n" - << " IsFinalized = " << (KV.second.IsFinalized ? "true" : "false") + << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") << "\n" << " " << KV.second.PendingQueries.size() << " pending queries: { "; @@ -1510,19 +1437,19 @@ void VSO::dump(raw_ostream &OS) { OS << "}\n Dependants:\n"; for (auto &KV2 : KV.second.Dependants) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; - OS << " Unfinalized Dependencies:\n"; - for (auto &KV2 : KV.second.UnfinalizedDependencies) + OS << " Unemitted Dependencies:\n"; + for (auto &KV2 : KV.second.UnemittedDependencies) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; } }); } -VSO::VSO(ExecutionSessionBase &ES, std::string Name) - : ES(ES), VSOName(std::move(Name)) { - SearchOrder.push_back(this); +JITDylib::JITDylib(ExecutionSession &ES, std::string Name) + : ES(ES), JITDylibName(std::move(Name)) { + SearchOrder.push_back({this, true}); } -Error VSO::defineImpl(MaterializationUnit &MU) { +Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; SymbolNameSet MUDefsOverridden; @@ -1599,8 +1526,8 @@ Error VSO::defineImpl(MaterializationUnit &MU) { return Error::success(); } -void VSO::detachQueryHelper(AsynchronousSymbolQuery &Q, - const SymbolNameSet &QuerySymbols) { +void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols) { for (auto &QuerySymbol : QuerySymbols) { assert(MaterializingInfos.count(QuerySymbol) && "QuerySymbol does not have MaterializingInfo"); @@ -1619,53 +1546,395 @@ void VSO::detachQueryHelper(AsynchronousSymbolQuery &Q, } } -void VSO::transferFinalizedNodeDependencies( +void JITDylib::transferEmittedNodeDependencies( MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName, - MaterializingInfo &FinalizedMI) { - for (auto &KV : FinalizedMI.UnfinalizedDependencies) { - auto &DependencyVSO = *KV.first; - SymbolNameSet *UnfinalizedDependenciesOnDependencyVSO = nullptr; + MaterializingInfo &EmittedMI) { + for (auto &KV : EmittedMI.UnemittedDependencies) { + auto &DependencyJD = *KV.first; + SymbolNameSet *UnemittedDependenciesOnDependencyJD = nullptr; for (auto &DependencyName : KV.second) { - auto &DependencyMI = DependencyVSO.MaterializingInfos[DependencyName]; + auto &DependencyMI = DependencyJD.MaterializingInfos[DependencyName]; // Do not add self dependencies. if (&DependencyMI == &DependantMI) continue; - // If we haven't looked up the dependencies for DependencyVSO yet, do it + // If we haven't looked up the dependencies for DependencyJD yet, do it // now and cache the result. - if (!UnfinalizedDependenciesOnDependencyVSO) - UnfinalizedDependenciesOnDependencyVSO = - &DependantMI.UnfinalizedDependencies[&DependencyVSO]; + if (!UnemittedDependenciesOnDependencyJD) + UnemittedDependenciesOnDependencyJD = + &DependantMI.UnemittedDependencies[&DependencyJD]; DependencyMI.Dependants[this].insert(DependantName); - UnfinalizedDependenciesOnDependencyVSO->insert(DependencyName); + UnemittedDependenciesOnDependencyJD->insert(DependencyName); } } } -VSO &ExecutionSession::createVSO(std::string Name) { - return runSessionLocked([&, this]() -> VSO & { - VSOs.push_back(std::unique_ptr<VSO>(new VSO(*this, std::move(Name)))); - return *VSOs.back(); +ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { + // Construct the main dylib. + JDs.push_back(std::unique_ptr<JITDylib>(new JITDylib(*this, "<main>"))); +} + +JITDylib &ExecutionSession::getMainJITDylib() { + return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); +} + +JITDylib &ExecutionSession::createJITDylib(std::string Name, + bool AddToMainDylibSearchOrder) { + return runSessionLocked([&, this]() -> JITDylib & { + JDs.push_back( + std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); + if (AddToMainDylibSearchOrder) + JDs.front()->addToSearchOrder(*JDs.back()); + return *JDs.back(); }); } -Expected<SymbolMap> lookup(const VSOList &VSOs, SymbolNameSet Names) { +void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { + assert(!!Err && "Error should be in failure state"); - if (VSOs.empty()) - return SymbolMap(); + bool SendErrorToQuery; + runSessionLocked([&]() { + Q.detach(); + SendErrorToQuery = Q.canStillFail(); + }); - auto &ES = (*VSOs.begin())->getExecutionSession(); + if (SendErrorToQuery) + Q.handleFailed(std::move(Err)); + else + reportError(std::move(Err)); +} - return ES.lookup(VSOs, Names, NoDependenciesToRegister, true); +Expected<SymbolMap> ExecutionSession::legacyLookup( + LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, + bool WaitUntilReady, RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } +#endif + + auto Query = std::make_shared<AsynchronousSymbolQuery>( + Names, std::move(OnResolve), std::move(OnReady)); + // FIXME: This should be run session locked along with the registration code + // and error reporting below. + SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); + + // If the query was lodged successfully then register the dependencies, + // otherwise fail it with an error. + if (UnresolvedSymbols.empty()) + RegisterDependencies(Query->QueryRegistrations); + else { + bool DeliverError = runSessionLocked([&]() { + Query->detach(); + return Query->canStillFail(); + }); + auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); + if (DeliverError) + Query->handleFailed(std::move(Err)); + else + reportError(std::move(Err)); + } + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif } -/// Look up a symbol by searching a list of VSOs. -Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name) { +void ExecutionSession::lookup( + const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, + SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + RegisterDependenciesFunction RegisterDependencies) { + + // lookup can be re-entered recursively if running on a single thread. Run any + // outstanding MUs in case this query depends on them, otherwise this lookup + // will starve waiting for a result from an MU that is stuck in the queue. + runOutstandingMUs(); + + auto Unresolved = std::move(Symbols); + std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; + auto Q = std::make_shared<AsynchronousSymbolQuery>( + Unresolved, std::move(OnResolve), std::move(OnReady)); + bool QueryIsFullyResolved = false; + bool QueryIsFullyReady = false; + bool QueryFailed = false; + + runSessionLocked([&]() { + for (auto &KV : SearchOrder) { + assert(KV.first && "JITDylibList entries must not be null"); + assert(!CollectedMUsMap.count(KV.first) && + "JITDylibList should not contain duplicate entries"); + + auto &JD = *KV.first; + auto MatchNonExported = KV.second; + JD.lodgeQuery(Q, Unresolved, MatchNonExported, CollectedMUsMap[&JD]); + } + + if (Unresolved.empty()) { + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryIsFullyResolved = Q->isFullyResolved(); + QueryIsFullyReady = Q->isFullyReady(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + } else { + // Query failed due to unresolved symbols. + QueryFailed = true; + + // Disconnect the query from its dependencies. + Q->detach(); + + // Replace the MUs. + for (auto &KV : CollectedMUsMap) + for (auto &MU : KV.second) + KV.first->replace(std::move(MU)); + } + }); + + if (QueryFailed) { + Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); + return; + } else { + if (QueryIsFullyResolved) + Q->handleFullyResolved(); + if (QueryIsFullyReady) + Q->handleFullyReady(); + } + + // Move the MUs to the OutstandingMUs list, then materialize. + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + for (auto &KV : CollectedMUsMap) + for (auto &MU : KV.second) + OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); + } + + runOutstandingMUs(); +} + +Expected<SymbolMap> ExecutionSession::lookup( + const JITDylibSearchList &SearchOrder, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, bool WaitUntilReady) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } +#endif + + // Perform the asynchronous lookup. + lookup(SearchOrder, Symbols, OnResolve, OnReady, RegisterDependencies); + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + SymbolStringPtr Name) { SymbolNameSet Names({Name}); - if (auto ResultMap = lookup(VSOs, std::move(Names))) { + + if (auto ResultMap = lookup(SearchOrder, std::move(Names), + NoDependenciesToRegister, true)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); @@ -1673,8 +1942,53 @@ Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name) { return ResultMap.takeError(); } -MangleAndInterner::MangleAndInterner(ExecutionSessionBase &ES, - const DataLayout &DL) +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, + SymbolStringPtr Name) { + SymbolNameSet Names({Name}); + + JITDylibSearchList FullSearchOrder; + FullSearchOrder.reserve(SearchOrder.size()); + for (auto *JD : SearchOrder) + FullSearchOrder.push_back({JD, false}); + + return lookup(FullSearchOrder, Name); +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name) { + return lookup(SearchOrder, intern(Name)); +} + +void ExecutionSession::dump(raw_ostream &OS) { + runSessionLocked([this, &OS]() { + for (auto &JD : JDs) + JD->dump(OS); + }); +} + +void ExecutionSession::runOutstandingMUs() { + while (1) { + std::pair<JITDylib *, std::unique_ptr<MaterializationUnit>> JITDylibAndMU; + + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + if (!OutstandingMUs.empty()) { + JITDylibAndMU = std::move(OutstandingMUs.back()); + OutstandingMUs.pop_back(); + } + } + + if (JITDylibAndMU.first) { + assert(JITDylibAndMU.second && "JITDylib, but no MU?"); + dispatchMaterialization(*JITDylibAndMU.first, + std::move(JITDylibAndMU.second)); + } else + break; + } +} + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) : ES(ES), DL(DL) {} SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { @@ -1683,7 +1997,7 @@ SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { raw_string_ostream MangledNameStream(MangledName); Mangler::getNameWithPrefix(MangledNameStream, Name, DL); } - return ES.getSymbolStringPool().intern(MangledName); + return ES.intern(MangledName); } } // End namespace orc. diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 6157677ce355..7c3c50b4d6e5 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -19,45 +19,6 @@ namespace llvm { namespace orc { -JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) - : TT(std::move(TT)) {} - -Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { - return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); -} - -Expected<std::unique_ptr<TargetMachine>> -JITTargetMachineBuilder::createTargetMachine() { - if (!Arch.empty()) { - Triple::ArchType Type = Triple::getArchTypeForLLVMName(Arch); - - if (Type == Triple::UnknownArch) - return make_error<StringError>(std::string("Unknown arch: ") + Arch, - inconvertibleErrorCode()); - } - - std::string ErrMsg; - auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); - if (!TheTarget) - return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - - auto *TM = - TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), - Options, RM, CM, OptLevel, /*JIT*/ true); - if (!TM) - return make_error<StringError>("Could not allocate target machine", - inconvertibleErrorCode()); - - return std::unique_ptr<TargetMachine>(TM); -} - -JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( - const std::vector<std::string> &FeatureVec) { - for (const auto &F : FeatureVec) - Features.AddFeature(F); - return *this; -} - CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) : InitList( GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), @@ -126,18 +87,24 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) { CtorDtorIterator(DtorsList, true)); } -void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) { - if (CtorDtors.begin() == CtorDtors.end()) +void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { + if (empty(CtorDtors)) return; MangleAndInterner Mangle( - V.getExecutionSession(), + JD.getExecutionSession(), (*CtorDtors.begin()).Func->getParent()->getDataLayout()); for (const auto &CtorDtor : CtorDtors) { assert(CtorDtor.Func && CtorDtor.Func->hasName() && "Ctor/Dtor function must be named to be runnable under the JIT"); + // FIXME: Maybe use a symbol promoter here instead. + if (CtorDtor.Func->hasLocalLinkage()) { + CtorDtor.Func->setLinkage(GlobalValue::ExternalLinkage); + CtorDtor.Func->setVisibility(GlobalValue::HiddenVisibility); + } + if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) { dbgs() << " Skipping because why now?\n"; continue; @@ -148,7 +115,7 @@ void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) { } } -Error CtorDtorRunner2::run() { +Error CtorDtorRunner::run() { using CtorDtorTy = void (*)(); SymbolNameSet Names; @@ -161,7 +128,10 @@ Error CtorDtorRunner2::run() { } } - if (auto CtorDtorMap = lookup({&V}, std::move(Names))) { + auto &ES = JD.getExecutionSession(); + if (auto CtorDtorMap = + ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names), + NoDependenciesToRegister, true)) { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); @@ -195,32 +165,46 @@ int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, return 0; } -Error LocalCXXRuntimeOverrides2::enable(VSO &V, MangleAndInterner &Mangle) { - SymbolMap RuntimeInterposes( - {{Mangle("__dso_handle"), - JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), - JITSymbolFlags::Exported)}, - {Mangle("__cxa_atexit"), - JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), - JITSymbolFlags::Exported)}}); +Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, + MangleAndInterner &Mangle) { + SymbolMap RuntimeInterposes; + RuntimeInterposes[Mangle("__dso_handle")] = + JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), + JITSymbolFlags::Exported); + RuntimeInterposes[Mangle("__cxa_atexit")] = + JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), + JITSymbolFlags::Exported); - return V.define(absoluteSymbols(std::move(RuntimeInterposes))); + return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } -DynamicLibraryFallbackGenerator::DynamicLibraryFallbackGenerator( +DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), GlobalPrefix(DL.getGlobalPrefix()) {} -SymbolNameSet DynamicLibraryFallbackGenerator:: -operator()(VSO &V, const SymbolNameSet &Names) { +Expected<DynamicLibrarySearchGenerator> +DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, + SymbolPredicate Allow) { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); + if (!Lib.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); +} + +SymbolNameSet DynamicLibrarySearchGenerator:: +operator()(JITDylib &JD, const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolMap NewSymbols; bool HasGlobalPrefix = (GlobalPrefix != '\0'); for (auto &Name : Names) { - if (!Allow(Name) || (*Name).empty()) + if ((*Name).empty()) + continue; + + if (Allow && !Allow(Name)) continue; if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) @@ -235,11 +219,11 @@ operator()(VSO &V, const SymbolNameSet &Names) { } } - // Add any new symbols to V. Since the fallback generator is only called for - // symbols that are not already defined, this will never trigger a duplicate + // Add any new symbols to JD. Since the generator is only called for symbols + // that are not already defined, this will never trigger a duplicate // definition error, so we can wrap this call in a 'cantFail'. if (!NewSymbols.empty()) - cantFail(V.define(absoluteSymbols(std::move(NewSymbols)))); + cantFail(JD.define(absoluteSymbols(std::move(NewSymbols)))); return Added; } diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index 0c17f9b7ad49..d952d1be70da 100644 --- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -12,28 +12,28 @@ namespace llvm { namespace orc { -IRCompileLayer2::IRCompileLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer, +IRCompileLayer::IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, CompileFunction Compile) : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {} -void IRCompileLayer2::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { +void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { std::lock_guard<std::mutex> Lock(IRLayerMutex); this->NotifyCompiled = std::move(NotifyCompiled); } -void IRCompileLayer2::emit(MaterializationResponsibility R, VModuleKey K, - std::unique_ptr<Module> M) { - assert(M && "Module must not be null"); +void IRCompileLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Module must not be null"); - if (auto Obj = Compile(*M)) { + if (auto Obj = Compile(*TSM.getModule())) { { std::lock_guard<std::mutex> Lock(IRLayerMutex); if (NotifyCompiled) - NotifyCompiled(K, std::move(M)); + NotifyCompiled(R.getVModuleKey(), std::move(TSM)); else - M = nullptr; + TSM = ThreadSafeModule(); } - BaseLayer.emit(std::move(R), std::move(K), std::move(*Obj)); + BaseLayer.emit(std::move(R), std::move(*Obj)); } else { R.failMaterialization(); getExecutionSession().reportError(Obj.takeError()); diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 4dd3cfdfe387..7bc0d696e3ac 100644 --- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -13,20 +13,20 @@ namespace llvm { namespace orc { -IRTransformLayer2::IRTransformLayer2(ExecutionSession &ES, +IRTransformLayer::IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, TransformFunction Transform) : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} -void IRTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, - std::unique_ptr<Module> M) { - assert(M && "Module must not be null"); +void IRTransformLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Module must not be null"); - if (auto TransformedMod = Transform(std::move(M))) - BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedMod)); + if (auto TransformedTSM = Transform(std::move(TSM), R)) + BaseLayer.emit(std::move(R), std::move(*TransformedTSM)); else { R.failMaterialization(); - getExecutionSession().reportError(TransformedMod.takeError()); + getExecutionSession().reportError(TransformedTSM.takeError()); } } diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 9ca2c5cb4a55..82000ec5b32b 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -27,19 +27,22 @@ public: using CompileFunction = JITCompileCallbackManager::CompileFunction; CompileCallbackMaterializationUnit(SymbolStringPtr Name, - CompileFunction Compile) - : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}})), + CompileFunction Compile, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), + std::move(K)), Name(std::move(Name)), Compile(std::move(Compile)) {} + StringRef getName() const override { return "<Compile Callbacks>"; } + private: - void materialize(MaterializationResponsibility R) { + void materialize(MaterializationResponsibility R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); R.resolve(Result); - R.finalize(); + R.emit(); } - void discard(const VSO &V, SymbolStringPtr Name) { + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { llvm_unreachable("Discard should never occur on a LMU?"); } @@ -52,20 +55,21 @@ private: namespace llvm { namespace orc { -void JITCompileCallbackManager::anchor() {} void IndirectStubsManager::anchor() {} +void TrampolinePool::anchor() {} Expected<JITTargetAddress> JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { - if (auto TrampolineAddr = getAvailableTrampolineAddr()) { - auto CallbackName = ES.getSymbolStringPool().intern( - std::string("cc") + std::to_string(++NextCallbackId)); + if (auto TrampolineAddr = TP->getTrampoline()) { + auto CallbackName = + ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); std::lock_guard<std::mutex> Lock(CCMgrMutex); AddrToSymbol[*TrampolineAddr] = CallbackName; - cantFail(CallbacksVSO.define( + cantFail(CallbacksJD.define( llvm::make_unique<CompileCallbackMaterializationUnit>( - std::move(CallbackName), std::move(Compile)))); + std::move(CallbackName), std::move(Compile), + ES.allocateVModule()))); return *TrampolineAddr; } else return TrampolineAddr.takeError(); @@ -88,7 +92,7 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( { raw_string_ostream ErrMsgStream(ErrMsg); ErrMsgStream << "No compile callback for trampoline at " - << format("0x%016x", TrampolineAddr); + << format("0x%016" PRIx64, TrampolineAddr); } ES.reportError( make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); @@ -97,9 +101,10 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( Name = I->second; } - if (auto Sym = lookup({&CallbacksVSO}, Name)) + if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name)) return Sym->getAddress(); else { + llvm::dbgs() << "Didn't find callback.\n"; // If anything goes wrong materializing Sym then report it to the session // and return the ErrorHandlerAddress; ES.reportError(Sym.takeError()); @@ -107,29 +112,46 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( } } -std::unique_ptr<JITCompileCallbackManager> +Expected<std::unique_ptr<JITCompileCallbackManager>> createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { switch (T.getArch()) { - default: return nullptr; - - case Triple::aarch64: { - typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; - return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + case Triple::aarch64: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); } case Triple::x86: { typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; - return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + case Triple::mipsel: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips64: + case Triple::mips64el: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); } case Triple::x86_64: { if ( T.getOS() == Triple::OSType::Win32 ) { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; - return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); + return CCMgrT::Create(ES, ErrorHandlerAddress); } else { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; - return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); + return CCMgrT::Create(ES, ErrorHandlerAddress); } } @@ -157,6 +179,25 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { orc::LocalIndirectStubsManager<orc::OrcI386>>(); }; + case Triple::mips: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); + }; + + case Triple::mipsel: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); + }; + + case Triple::mips64: + case Triple::mips64el: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips64>>(); + }; + case Triple::x86_64: if (T.getOS() == Triple::OSType::Win32) { return [](){ @@ -210,57 +251,34 @@ void makeStub(Function &F, Value &ImplPointer) { Builder.CreateRet(Call); } -// Utility class for renaming global values and functions during partitioning. -class GlobalRenamer { -public: - - static bool needsRenaming(const Value &New) { - return !New.hasName() || New.getName().startswith("\01L"); - } - - const std::string& getRename(const Value &Orig) { - // See if we have a name for this global. - { - auto I = Names.find(&Orig); - if (I != Names.end()) - return I->second; +std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { + std::vector<GlobalValue *> PromotedGlobals; + + for (auto &GV : M.global_values()) { + bool Promoted = true; + + // Rename if necessary. + if (!GV.hasName()) + GV.setName("__orc_anon." + Twine(NextId++)); + else if (GV.getName().startswith("\01L")) + GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); + else if (GV.hasLocalLinkage()) + GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); + else + Promoted = false; + + if (GV.hasLocalLinkage()) { + GV.setLinkage(GlobalValue::ExternalLinkage); + GV.setVisibility(GlobalValue::HiddenVisibility); + Promoted = true; } + GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); - // Nope. Create a new one. - // FIXME: Use a more robust uniquing scheme. (This may blow up if the user - // writes a "__orc_anon[[:digit:]]* method). - unsigned ID = Names.size(); - std::ostringstream NameStream; - NameStream << "__orc_anon" << ID++; - auto I = Names.insert(std::make_pair(&Orig, NameStream.str())); - return I.first->second; + if (Promoted) + PromotedGlobals.push_back(&GV); } -private: - DenseMap<const Value*, std::string> Names; -}; - -static void raiseVisibilityOnValue(GlobalValue &V, GlobalRenamer &R) { - if (V.hasLocalLinkage()) { - if (R.needsRenaming(V)) - V.setName(R.getRename(V)); - V.setLinkage(GlobalValue::ExternalLinkage); - V.setVisibility(GlobalValue::HiddenVisibility); - } - V.setUnnamedAddr(GlobalValue::UnnamedAddr::None); - assert(!R.needsRenaming(V) && "Invalid global name."); -} - -void makeAllSymbolsExternallyAccessible(Module &M) { - GlobalRenamer Renamer; - - for (auto &F : M) - raiseVisibilityOnValue(F, Renamer); - - for (auto &GV : M.globals()) - raiseVisibilityOnValue(GV, Renamer); - for (auto &A : M.aliases()) - raiseVisibilityOnValue(A, Renamer); + return PromotedGlobals; } Function* cloneFunctionDecl(Module &Dst, const Function &F, diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp new file mode 100644 index 000000000000..4af09d196ff9 --- /dev/null +++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -0,0 +1,55 @@ +//===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" + +#include "llvm/Support/TargetRegistry.h" + +namespace llvm { +namespace orc { + +JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) + : TT(std::move(TT)) { + Options.EmulatedTLS = true; + Options.ExplicitEmulatedTLS = true; +} + +Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { + // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, + // rather than a valid triple for the current process. + return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); +} + +Expected<std::unique_ptr<TargetMachine>> +JITTargetMachineBuilder::createTargetMachine() { + + std::string ErrMsg; + auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); + if (!TheTarget) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + auto *TM = + TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), + Options, RM, CM, OptLevel, /*JIT*/ true); + if (!TM) + return make_error<StringError>("Could not allocate target machine", + inconvertibleErrorCode()); + + return std::unique_ptr<TargetMachine>(TM); +} + +JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( + const std::vector<std::string> &FeatureVec) { + for (const auto &F : FeatureVec) + Features.AddFeature(F); + return *this; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp index 52ff4efe56b2..e2089f9106bd 100644 --- a/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -12,49 +12,109 @@ #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/Mangler.h" +namespace { + + // A SimpleCompiler that owns its TargetMachine. + class TMOwningSimpleCompiler : public llvm::orc::SimpleCompiler { + public: + TMOwningSimpleCompiler(std::unique_ptr<llvm::TargetMachine> TM) + : llvm::orc::SimpleCompiler(*TM), TM(std::move(TM)) {} + private: + // FIXME: shared because std::functions (and thus + // IRCompileLayer::CompileFunction) are not moveable. + std::shared_ptr<llvm::TargetMachine> TM; + }; + +} // end anonymous namespace + namespace llvm { namespace orc { +LLJIT::~LLJIT() { + if (CompileThreads) + CompileThreads->wait(); +} + Expected<std::unique_ptr<LLJIT>> -LLJIT::Create(std::unique_ptr<ExecutionSession> ES, - std::unique_ptr<TargetMachine> TM, DataLayout DL) { - return std::unique_ptr<LLJIT>( - new LLJIT(std::move(ES), std::move(TM), std::move(DL))); +LLJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, + unsigned NumCompileThreads) { + + if (NumCompileThreads == 0) { + // If NumCompileThreads == 0 then create a single-threaded LLJIT instance. + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), + std::move(*TM), std::move(DL))); + } + + return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), + std::move(JTMB), std::move(DL), + NumCompileThreads)); } Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { - auto InternedName = ES->getSymbolStringPool().intern(Name); + auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); return Main.define(absoluteSymbols(std::move(Symbols))); } -Error LLJIT::addIRModule(VSO &V, std::unique_ptr<Module> M) { - assert(M && "Can not add null module"); +Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); - if (auto Err = applyDataLayout(*M)) + if (auto Err = applyDataLayout(*TSM.getModule())) return Err; - auto K = ES->allocateVModule(); - return CompileLayer.add(V, K, std::move(M)); + return CompileLayer.add(JD, std::move(TSM), ES->allocateVModule()); +} + +Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { + assert(Obj && "Can not add null object"); + + return ObjLinkingLayer.add(JD, std::move(Obj), ES->allocateVModule()); } -Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(VSO &V, +Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, StringRef Name) { - return llvm::orc::lookup({&V}, ES->getSymbolStringPool().intern(Name)); + return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); } LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, DataLayout DL) - : ES(std::move(ES)), Main(this->ES->createVSO("main")), TM(std::move(TM)), - DL(std::move(DL)), - ObjLinkingLayer(*this->ES, - [this](VModuleKey K) { return getMemoryManager(K); }), - CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)), + : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), + ObjLinkingLayer( + *this->ES, + []() { return llvm::make_unique<SectionMemoryManager>(); }), + CompileLayer(*this->ES, ObjLinkingLayer, + TMOwningSimpleCompiler(std::move(TM))), CtorRunner(Main), DtorRunner(Main) {} -std::shared_ptr<RuntimeDyld::MemoryManager> -LLJIT::getMemoryManager(VModuleKey K) { - return llvm::make_unique<SectionMemoryManager>(); +LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, + DataLayout DL, unsigned NumCompileThreads) + : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), + ObjLinkingLayer( + *this->ES, + []() { return llvm::make_unique<SectionMemoryManager>(); }), + CompileLayer(*this->ES, ObjLinkingLayer, + ConcurrentIRCompiler(std::move(JTMB))), + CtorRunner(Main), DtorRunner(Main) { + assert(NumCompileThreads != 0 && + "Multithreaded LLJIT instance can not be created with 0 threads"); + + // Move modules to new contexts when they're emitted so that we can compile + // them in parallel. + CompileLayer.setCloneToNewContextOnEmit(true); + + // Create a thread pool to compile on and set the execution session + // dispatcher to use the thread pool. + CompileThreads = llvm::make_unique<ThreadPool>(NumCompileThreads); + this->ES->setDispatchMaterialization( + [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { + // FIXME: Switch to move capture once we have c++14. + auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); + auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + CompileThreads->async(std::move(Work)); + }); } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -84,16 +144,15 @@ void LLJIT::recordCtorDtors(Module &M) { } Expected<std::unique_ptr<LLLazyJIT>> -LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES, - std::unique_ptr<TargetMachine> TM, DataLayout DL, - LLVMContext &Ctx) { - const Triple &TT = TM->getTargetTriple(); +LLLazyJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, + JITTargetAddress ErrorAddr, unsigned NumCompileThreads) { + auto ES = llvm::make_unique<ExecutionSession>(); - auto CCMgr = createLocalCompileCallbackManager(TT, *ES, 0); - if (!CCMgr) - return make_error<StringError>( - std::string("No callback manager available for ") + TT.str(), - inconvertibleErrorCode()); + const Triple &TT = JTMB.getTargetTriple(); + + auto LCTMgr = createLocalLazyCallThroughManager(TT, *ES, ErrorAddr); + if (!LCTMgr) + return LCTMgr.takeError(); auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); if (!ISMBuilder) @@ -101,34 +160,51 @@ LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES, std::string("No indirect stubs manager builder for ") + TT.str(), inconvertibleErrorCode()); - return std::unique_ptr<LLLazyJIT>( - new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL), Ctx, - std::move(CCMgr), std::move(ISMBuilder))); + if (NumCompileThreads == 0) { + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + return std::unique_ptr<LLLazyJIT>( + new LLLazyJIT(std::move(ES), std::move(*TM), std::move(DL), + std::move(*LCTMgr), std::move(ISMBuilder))); + } + + return std::unique_ptr<LLLazyJIT>(new LLLazyJIT( + std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads, + std::move(*LCTMgr), std::move(ISMBuilder))); } -Error LLLazyJIT::addLazyIRModule(VSO &V, std::unique_ptr<Module> M) { - assert(M && "Can not add null module"); +Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); - if (auto Err = applyDataLayout(*M)) + if (auto Err = applyDataLayout(*TSM.getModule())) return Err; - makeAllSymbolsExternallyAccessible(*M); + recordCtorDtors(*TSM.getModule()); - recordCtorDtors(*M); - - auto K = ES->allocateVModule(); - return CODLayer.add(V, K, std::move(M)); + return CODLayer.add(JD, std::move(TSM), ES->allocateVModule()); } LLLazyJIT::LLLazyJIT( std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, - DataLayout DL, LLVMContext &Ctx, - std::unique_ptr<JITCompileCallbackManager> CCMgr, + DataLayout DL, std::unique_ptr<LazyCallThroughManager> LCTMgr, std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) : LLJIT(std::move(ES), std::move(TM), std::move(DL)), - CCMgr(std::move(CCMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->CCMgr, std::move(ISMBuilder), - [&]() -> LLVMContext & { return Ctx; }) {} + LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), + CODLayer(*this->ES, TransformLayer, *this->LCTMgr, + std::move(ISMBuilder)) {} + +LLLazyJIT::LLLazyJIT( + std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, + DataLayout DL, unsigned NumCompileThreads, + std::unique_ptr<LazyCallThroughManager> LCTMgr, + std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) + : LLJIT(std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads), + LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), + CODLayer(*this->ES, TransformLayer, *this->LCTMgr, + std::move(ISMBuilder)) { + CODLayer.setCloneToNewContextOnEmit(true); +} } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp index b9da3b7fb8d5..11af76825e9f 100644 --- a/lib/ExecutionEngine/Orc/Layer.cpp +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -9,7 +9,9 @@ #include "llvm/ExecutionEngine/Orc/Layer.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" namespace llvm { namespace orc { @@ -17,17 +19,19 @@ namespace orc { IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} IRLayer::~IRLayer() {} -Error IRLayer::add(VSO &V, VModuleKey K, std::unique_ptr<Module> M) { - return V.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( - *this, std::move(K), std::move(M))); +Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { + return JD.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( + *this, std::move(K), std::move(TSM))); } IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, - std::unique_ptr<Module> M) - : MaterializationUnit(SymbolFlagsMap()), M(std::move(M)) { + ThreadSafeModule TSM, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap(), std::move(K)), TSM(std::move(TSM)) { + + assert(this->TSM && "Module must not be null"); - MangleAndInterner Mangle(ES, this->M->getDataLayout()); - for (auto &G : this->M->global_values()) { + MangleAndInterner Mangle(ES, this->TSM.getModule()->getDataLayout()); + for (auto &G : this->TSM.getModule()->global_values()) { if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { auto MangledName = Mangle(G.getName()); @@ -38,12 +42,24 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, } IRMaterializationUnit::IRMaterializationUnit( - std::unique_ptr<Module> M, SymbolFlagsMap SymbolFlags, + ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, SymbolNameToDefinitionMap SymbolToDefinition) - : MaterializationUnit(std::move(SymbolFlags)), M(std::move(M)), - SymbolToDefinition(std::move(SymbolToDefinition)) {} + : MaterializationUnit(std::move(SymbolFlags), std::move(K)), + TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} + +StringRef IRMaterializationUnit::getName() const { + if (TSM.getModule()) + return TSM.getModule()->getModuleIdentifier(); + return "<null module>"; +} + +void IRMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD.getName() << " discarding " << *Name << " from MU@" + << this << " (" << getName() << ")\n"; + });); -void IRMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { auto I = SymbolToDefinition.find(Name); assert(I != SymbolToDefinition.end() && "Symbol not provided by this MU, or previously discarded"); @@ -54,53 +70,117 @@ void IRMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { } BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( - IRLayer &L, VModuleKey K, std::unique_ptr<Module> M) - : IRMaterializationUnit(L.getExecutionSession(), std::move(M)), + IRLayer &L, VModuleKey K, ThreadSafeModule TSM) + : IRMaterializationUnit(L.getExecutionSession(), std::move(TSM), + std::move(K)), L(L), K(std::move(K)) {} void BasicIRLayerMaterializationUnit::materialize( MaterializationResponsibility R) { - L.emit(std::move(R), std::move(K), std::move(M)); + + // Throw away the SymbolToDefinition map: it's not usable after we hand + // off the module. + SymbolToDefinition.clear(); + + // If cloneToNewContextOnEmit is set, clone the module now. + if (L.getCloneToNewContextOnEmit()) + TSM = cloneToNewContext(TSM); + +#ifndef NDEBUG + auto &ES = R.getTargetJITDylib().getExecutionSession(); +#endif // NDEBUG + + auto Lock = TSM.getContextLock(); + LLVM_DEBUG(ES.runSessionLocked([&]() { + dbgs() << "Emitting, for " << R.getTargetJITDylib().getName() << ", " + << *this << "\n"; + });); + L.emit(std::move(R), std::move(TSM)); + LLVM_DEBUG(ES.runSessionLocked([&]() { + dbgs() << "Finished emitting, for " << R.getTargetJITDylib().getName() + << ", " << *this << "\n"; + });); } ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} ObjectLayer::~ObjectLayer() {} -Error ObjectLayer::add(VSO &V, VModuleKey K, std::unique_ptr<MemoryBuffer> O) { - return V.define(llvm::make_unique<BasicObjectLayerMaterializationUnit>( - *this, std::move(K), std::move(O))); +Error ObjectLayer::add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O, + VModuleKey K) { + auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(K), + std::move(O)); + if (!ObjMU) + return ObjMU.takeError(); + return JD.define(std::move(*ObjMU)); +} + +Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> +BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + auto SymbolFlags = + getObjectSymbolFlags(L.getExecutionSession(), O->getMemBufferRef()); + + if (!SymbolFlags) + return SymbolFlags.takeError(); + + return std::unique_ptr<BasicObjectLayerMaterializationUnit>( + new BasicObjectLayerMaterializationUnit(L, K, std::move(O), + std::move(*SymbolFlags))); } BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( - ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O) - : MaterializationUnit(SymbolFlagsMap()), L(L), K(std::move(K)), - O(std::move(O)) { - - auto &ES = L.getExecutionSession(); - auto Obj = cantFail( - object::ObjectFile::createObjectFile(this->O->getMemBufferRef())); - - for (auto &Sym : Obj->symbols()) { - if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) && - (Sym.getFlags() & object::BasicSymbolRef::SF_Exported)) { - auto InternedName = - ES.getSymbolStringPool().intern(cantFail(Sym.getName())); - SymbolFlags[InternedName] = JITSymbolFlags::fromObjectSymbol(Sym); - } - } + ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O, + SymbolFlagsMap SymbolFlags) + : MaterializationUnit(std::move(SymbolFlags), std::move(K)), L(L), + O(std::move(O)) {} + +StringRef BasicObjectLayerMaterializationUnit::getName() const { + if (O) + return O->getBufferIdentifier(); + return "<null object>"; } void BasicObjectLayerMaterializationUnit::materialize( MaterializationResponsibility R) { - L.emit(std::move(R), std::move(K), std::move(O)); + L.emit(std::move(R), std::move(O)); } -void BasicObjectLayerMaterializationUnit::discard(const VSO &V, - SymbolStringPtr Name) { +void BasicObjectLayerMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { // FIXME: Support object file level discard. This could be done by building a // filter to pass to the object layer along with the object itself. } +Expected<SymbolFlagsMap> getObjectSymbolFlags(ExecutionSession &ES, + MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + // Skip symbols not defined in this object file. + if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) + continue; + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + return SymbolFlags; +} + } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp new file mode 100644 index 000000000000..55f4a7c5afce --- /dev/null +++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -0,0 +1,208 @@ +//===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +void LazyCallThroughManager::NotifyResolvedFunction::anchor() {} + +LazyCallThroughManager::LazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, + std::unique_ptr<TrampolinePool> TP) + : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(std::move(TP)) {} + +Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( + JITDylib &SourceJD, SymbolStringPtr SymbolName, + std::shared_ptr<NotifyResolvedFunction> NotifyResolved) { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto Trampoline = TP->getTrampoline(); + + if (!Trampoline) + return Trampoline.takeError(); + + Reexports[*Trampoline] = std::make_pair(&SourceJD, std::move(SymbolName)); + Notifiers[*Trampoline] = std::move(NotifyResolved); + return *Trampoline; +} + +JITTargetAddress +LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { + JITDylib *SourceJD = nullptr; + SymbolStringPtr SymbolName; + + { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Reexports.find(TrampolineAddr); + if (I == Reexports.end()) + return ErrorHandlerAddr; + SourceJD = I->second.first; + SymbolName = I->second.second; + } + + auto LookupResult = ES.lookup(JITDylibSearchList({{SourceJD, true}}), + {SymbolName}, NoDependenciesToRegister, true); + + if (!LookupResult) { + ES.reportError(LookupResult.takeError()); + return ErrorHandlerAddr; + } + + assert(LookupResult->size() == 1 && "Unexpected number of results"); + assert(LookupResult->count(SymbolName) && "Unexpected result"); + + auto ResolvedAddr = LookupResult->begin()->second.getAddress(); + + std::shared_ptr<NotifyResolvedFunction> NotifyResolved = nullptr; + { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Notifiers.find(TrampolineAddr); + if (I != Notifiers.end()) { + NotifyResolved = I->second; + Notifiers.erase(I); + } + } + + if (NotifyResolved) { + if (auto Err = (*NotifyResolved)(*SourceJD, SymbolName, ResolvedAddr)) { + ES.reportError(std::move(Err)); + return ErrorHandlerAddr; + } + } + + return ResolvedAddr; +} + +Expected<std::unique_ptr<LazyCallThroughManager>> +createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr) { + switch (T.getArch()) { + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + + case Triple::aarch64: + return LocalLazyCallThroughManager::Create<OrcAArch64>(ES, + ErrorHandlerAddr); + + case Triple::x86: + return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr); + + case Triple::mips: + return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES, + ErrorHandlerAddr); + + case Triple::mipsel: + return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES, + ErrorHandlerAddr); + + case Triple::mips64: + case Triple::mips64el: + return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr); + + case Triple::x86_64: + if (T.getOS() == Triple::OSType::Win32) + return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>( + ES, ErrorHandlerAddr); + else + return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>( + ES, ErrorHandlerAddr); + } +} + +LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( + LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, + JITDylib &SourceJD, SymbolAliasMap CallableAliases, VModuleKey K) + : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), + LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), + CallableAliases(std::move(CallableAliases)), + NotifyResolved(LazyCallThroughManager::createNotifyResolvedFunction( + [&ISManager](JITDylib &JD, const SymbolStringPtr &SymbolName, + JITTargetAddress ResolvedAddr) { + return ISManager.updatePointer(*SymbolName, ResolvedAddr); + })) {} + +StringRef LazyReexportsMaterializationUnit::getName() const { + return "<Lazy Reexports>"; +} + +void LazyReexportsMaterializationUnit::materialize( + MaterializationResponsibility R) { + auto RequestedSymbols = R.getRequestedSymbols(); + + SymbolAliasMap RequestedAliases; + for (auto &RequestedSymbol : RequestedSymbols) { + auto I = CallableAliases.find(RequestedSymbol); + assert(I != CallableAliases.end() && "Symbol not found in alias map?"); + RequestedAliases[I->first] = std::move(I->second); + CallableAliases.erase(I); + } + + if (!CallableAliases.empty()) + R.replace(lazyReexports(LCTManager, ISManager, SourceJD, + std::move(CallableAliases))); + + IndirectStubsManager::StubInitsMap StubInits; + for (auto &Alias : RequestedAliases) { + + auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( + SourceJD, Alias.second.Aliasee, NotifyResolved); + + if (!CallThroughTrampoline) { + SourceJD.getExecutionSession().reportError( + CallThroughTrampoline.takeError()); + R.failMaterialization(); + return; + } + + StubInits[*Alias.first] = + std::make_pair(*CallThroughTrampoline, Alias.second.AliasFlags); + } + + if (auto Err = ISManager.createStubs(StubInits)) { + SourceJD.getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + SymbolMap Stubs; + for (auto &Alias : RequestedAliases) + Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); + + R.resolve(Stubs); + R.emit(); +} + +void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(CallableAliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + CallableAliases.erase(Name); +} + +SymbolFlagsMap +LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) { + assert(KV.second.AliasFlags.isCallable() && + "Lazy re-exports must be callable symbols"); + SymbolFlags[KV.first] = KV.second.AliasFlags; + } + return SymbolFlags; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp index 18be9a042f7f..ddb72544b770 100644 --- a/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/lib/ExecutionEngine/Orc/Legacy.cpp @@ -18,47 +18,47 @@ JITSymbolResolverAdapter::JITSymbolResolverAdapter( ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR) : ES(ES), R(R), MR(MR) {} -Expected<JITSymbolResolverAdapter::LookupResult> -JITSymbolResolverAdapter::lookup(const LookupSet &Symbols) { +void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, + OnResolvedFunction OnResolved) { SymbolNameSet InternedSymbols; for (auto &S : Symbols) - InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + InternedSymbols.insert(ES.intern(S)); - auto LookupFn = [&, this](std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Unresolved) { - return R.lookup(std::move(Q), std::move(Unresolved)); - }; + auto OnResolvedWithUnwrap = [OnResolved](Expected<SymbolMap> InternedResult) { + if (!InternedResult) { + OnResolved(InternedResult.takeError()); + return; + } - auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { - if (MR) - MR->addDependenciesForAll(Deps); + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + OnResolved(Result); }; - auto InternedResult = - ES.legacyLookup(ES, std::move(LookupFn), std::move(InternedSymbols), - false, RegisterDependencies); - - if (!InternedResult) - return InternedResult.takeError(); + auto Q = std::make_shared<AsynchronousSymbolQuery>( + InternedSymbols, OnResolvedWithUnwrap, + [this](Error Err) { ES.reportError(std::move(Err)); }); - JITSymbolResolver::LookupResult Result; - for (auto &KV : *InternedResult) - Result[*KV.first] = KV.second; - - return Result; + auto Unresolved = R.lookup(Q, InternedSymbols); + if (Unresolved.empty()) { + if (MR) + MR->addDependenciesForAll(Q->QueryRegistrations); + } else + ES.legacyFailQuery(*Q, make_error<SymbolsNotFound>(std::move(Unresolved))); } -Expected<JITSymbolResolverAdapter::LookupFlagsResult> -JITSymbolResolverAdapter::lookupFlags(const LookupSet &Symbols) { +Expected<JITSymbolResolverAdapter::LookupSet> +JITSymbolResolverAdapter::getResponsibilitySet(const LookupSet &Symbols) { SymbolNameSet InternedSymbols; for (auto &S : Symbols) - InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + InternedSymbols.insert(ES.intern(S)); - SymbolFlagsMap SymbolFlags = R.lookupFlags(InternedSymbols); - LookupFlagsResult Result; - for (auto &KV : SymbolFlags) { - ResolvedStrings.insert(KV.first); - Result[*KV.first] = KV.second; + auto InternedResult = R.getResponsibilitySet(InternedSymbols); + LookupSet Result; + for (auto &S : InternedResult) { + ResolvedStrings.insert(S); + Result.insert(*S); } return Result; diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp index 3796e3d37bc2..922fc6f021ce 100644 --- a/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -14,8 +14,8 @@ namespace llvm { namespace orc { -SymbolFlagsMap NullResolver::lookupFlags(const SymbolNameSet &Symbols) { - return SymbolFlagsMap(); +SymbolNameSet NullResolver::getResponsibilitySet(const SymbolNameSet &Symbols) { + return Symbols; } SymbolNameSet diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index 6980c8140fd0..825f53204736 100644 --- a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -13,17 +13,17 @@ namespace llvm { namespace orc { -ObjectTransformLayer2::ObjectTransformLayer2(ExecutionSession &ES, - ObjectLayer &BaseLayer, - TransformFunction Transform) +ObjectTransformLayer::ObjectTransformLayer(ExecutionSession &ES, + ObjectLayer &BaseLayer, + TransformFunction Transform) : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} -void ObjectTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, - std::unique_ptr<MemoryBuffer> O) { +void ObjectTransformLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { assert(O && "Module must not be null"); if (auto TransformedObj = Transform(std::move(O))) - BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedObj)); + BaseLayer.emit(std::move(R), std::move(*TransformedObj)); else { R.failMaterialization(); getExecutionSession().reportError(TransformedObj.takeError()); diff --git a/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/lib/ExecutionEngine/Orc/OrcABISupport.cpp index e3c968157976..aa4055542426 100644 --- a/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -537,5 +537,448 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, return Error::success(); } +void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, + JITReentryFn ReentryFn, + void *CallbackMgr, bool isBigEndian) { + + const uint32_t ResolverCode[] = { + // resolver_entry: + 0x27bdff98, // 0x00: addiu $sp,$sp,-104 + 0xafa20000, // 0x04: sw $v0,0($sp) + 0xafa30004, // 0x08: sw $v1,4($sp) + 0xafa40008, // 0x0c: sw $a0,8($sp) + 0xafa5000c, // 0x10: sw $a1,12($sp) + 0xafa60010, // 0x14: sw $a2,16($sp) + 0xafa70014, // 0x18: sw $a3,20($sp) + 0xafb00018, // 0x1c: sw $s0,24($sp) + 0xafb1001c, // 0x20: sw $s1,28($sp) + 0xafb20020, // 0x24: sw $s2,32($sp) + 0xafb30024, // 0x28: sw $s3,36($sp) + 0xafb40028, // 0x2c: sw $s4,40($sp) + 0xafb5002c, // 0x30: sw $s5,44($sp) + 0xafb60030, // 0x34: sw $s6,48($sp) + 0xafb70034, // 0x38: sw $s7,52($sp) + 0xafa80038, // 0x3c: sw $t0,56($sp) + 0xafa9003c, // 0x40: sw $t1,60($sp) + 0xafaa0040, // 0x44: sw $t2,64($sp) + 0xafab0044, // 0x48: sw $t3,68($sp) + 0xafac0048, // 0x4c: sw $t4,72($sp) + 0xafad004c, // 0x50: sw $t5,76($sp) + 0xafae0050, // 0x54: sw $t6,80($sp) + 0xafaf0054, // 0x58: sw $t7,84($sp) + 0xafb80058, // 0x5c: sw $t8,88($sp) + 0xafb9005c, // 0x60: sw $t9,92($sp) + 0xafbe0060, // 0x64: sw $fp,96($sp) + 0xafbf0064, // 0x68: sw $ra,100($sp) + + // Callback manager addr. + 0x00000000, // 0x6c: lui $a0,callbackmgr + 0x00000000, // 0x70: addiu $a0,$a0,callbackmgr + + 0x03e02825, // 0x74: move $a1, $ra + 0x24a5ffec, // 0x78: addiu $a1,$a1,-20 + + // JIT re-entry fn addr: + 0x00000000, // 0x7c: lui $t9,reentry + 0x00000000, // 0x80: addiu $t9,$t9,reentry + + 0x0320f809, // 0x84: jalr $t9 + 0x00000000, // 0x88: nop + 0x8fbf0064, // 0x8c: lw $ra,100($sp) + 0x8fbe0060, // 0x90: lw $fp,96($sp) + 0x8fb9005c, // 0x94: lw $t9,92($sp) + 0x8fb80058, // 0x98: lw $t8,88($sp) + 0x8faf0054, // 0x9c: lw $t7,84($sp) + 0x8fae0050, // 0xa0: lw $t6,80($sp) + 0x8fad004c, // 0xa4: lw $t5,76($sp) + 0x8fac0048, // 0xa8: lw $t4,72($sp) + 0x8fab0044, // 0xac: lw $t3,68($sp) + 0x8faa0040, // 0xb0: lw $t2,64($sp) + 0x8fa9003c, // 0xb4: lw $t1,60($sp) + 0x8fa80038, // 0xb8: lw $t0,56($sp) + 0x8fb70034, // 0xbc: lw $s7,52($sp) + 0x8fb60030, // 0xc0: lw $s6,48($sp) + 0x8fb5002c, // 0xc4: lw $s5,44($sp) + 0x8fb40028, // 0xc8: lw $s4,40($sp) + 0x8fb30024, // 0xcc: lw $s3,36($sp) + 0x8fb20020, // 0xd0: lw $s2,32($sp) + 0x8fb1001c, // 0xd4: lw $s1,28($sp) + 0x8fb00018, // 0xd8: lw $s0,24($sp) + 0x8fa70014, // 0xdc: lw $a3,20($sp) + 0x8fa60010, // 0xe0: lw $a2,16($sp) + 0x8fa5000c, // 0xe4: lw $a1,12($sp) + 0x8fa40008, // 0xe8: lw $a0,8($sp) + 0x27bd0068, // 0xec: addiu $sp,$sp,104 + 0x0300f825, // 0xf0: move $ra, $t8 + 0x03200008, // 0xf4: jr $t9 + 0x00000000, // 0xf8: move $t9, $v0/v1 + }; + + const unsigned ReentryFnAddrOffset = 0x7c; // JIT re-entry fn addr lui + const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui + const unsigned Offsett = 0xf8; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + + // Depending on endian return value will be in v0 or v1. + uint32_t MoveVxT9 = isBigEndian ? 0x0060c825 : 0x0040c825; + memcpy(ResolverMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); + + uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); + uint32_t CallMgrLUi = 0x3c040000 | (((CallMgrAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t CallMgrADDiu = 0x24840000 | ((CallMgrAddr) & 0xFFFF); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); + memcpy(ResolverMem + CallbackMgrAddrOffset + 4, &CallMgrADDiu, + sizeof(CallMgrADDiu)); + + uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); + uint32_t ReentryLUi = 0x3c190000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryADDiu = 0x27390000 | ((ReentryAddr) & 0xFFFF); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); + memcpy(ResolverMem + ReentryFnAddrOffset + 4, &ReentryADDiu, + sizeof(ReentryADDiu)); +} + +void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, + void *ResolverAddr, + unsigned NumTrampolines) { + + uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); + uint32_t RHiAddr = ((ResolveAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + Trampolines[5 * I + 0] = 0x03e0c025; // move $t8,$ra + Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); // lui $t9,resolveAddr + Trampolines[5 * I + 2] = 0x27390000 | (ResolveAddr & 0xFFFF); // addiu $t9,$t9,resolveAddr + Trampolines[5 * I + 3] = 0x0320f809; // jalr $t9 + Trampolines[5 * I + 4] = 0x00000000; // nop + } +} + +Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9, ptr1 + // lw $t9, %lo(ptr1)($t9) + // jr $t9 + // stub2: + // lui $t9, ptr2 + // lw $t9,%lo(ptr1)($t9) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .word 0x0 + // ptr2: + // .word 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + unsigned PageSize = sys::Process::getPageSize(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); + uint64_t PtrAddr = reinterpret_cast<uint64_t>(Stub) + NumPages * PageSize; + + for (unsigned I = 0; I < NumStubs; ++I) { + uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[4 * I + 0] = 0x3c190000 | (HiAddr & 0xFFFF); // lui $t9,ptr1 + Stub[4 * I + 1] = 0x8f390000 | (PtrAddr & 0xFFFF); // lw $t9,%lo(ptr1)($t9) + Stub[4 * I + 2] = 0x03200008; // jr $t9 + Stub[4 * I + 3] = 0x00000000; // nop + PtrAddr += 4; + } + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} + +void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint32_t ResolverCode[] = { + //resolver_entry: + 0x67bdff30, // 0x00: daddiu $sp,$sp,-208 + 0xffa20000, // 0x04: sd v0,0(sp) + 0xffa30008, // 0x08: sd v1,8(sp) + 0xffa40010, // 0x0c: sd a0,16(sp) + 0xffa50018, // 0x10: sd a1,24(sp) + 0xffa60020, // 0x14: sd a2,32(sp) + 0xffa70028, // 0x18: sd a3,40(sp) + 0xffa80030, // 0x1c: sd a4,48(sp) + 0xffa90038, // 0x20: sd a5,56(sp) + 0xffaa0040, // 0x24: sd a6,64(sp) + 0xffab0048, // 0x28: sd a7,72(sp) + 0xffac0050, // 0x2c: sd t0,80(sp) + 0xffad0058, // 0x30: sd t1,88(sp) + 0xffae0060, // 0x34: sd t2,96(sp) + 0xffaf0068, // 0x38: sd t3,104(sp) + 0xffb00070, // 0x3c: sd s0,112(sp) + 0xffb10078, // 0x40: sd s1,120(sp) + 0xffb20080, // 0x44: sd s2,128(sp) + 0xffb30088, // 0x48: sd s3,136(sp) + 0xffb40090, // 0x4c: sd s4,144(sp) + 0xffb50098, // 0x50: sd s5,152(sp) + 0xffb600a0, // 0x54: sd s6,160(sp) + 0xffb700a8, // 0x58: sd s7,168(sp) + 0xffb800b0, // 0x5c: sd t8,176(sp) + 0xffb900b8, // 0x60: sd t9,184(sp) + 0xffbe00c0, // 0x64: sd fp,192(sp) + 0xffbf00c8, // 0x68: sd ra,200(sp) + + // Callback manager addr. + 0x00000000, // 0x6c: lui $a0,heighest(callbackmgr) + 0x00000000, // 0x70: daddiu $a0,$a0,heigher(callbackmgr) + 0x00000000, // 0x74: dsll $a0,$a0,16 + 0x00000000, // 0x78: daddiu $a0,$a0,hi(callbackmgr) + 0x00000000, // 0x7c: dsll $a0,$a0,16 + 0x00000000, // 0x80: daddiu $a0,$a0,lo(callbackmgr) + + 0x03e02825, // 0x84: move $a1, $ra + 0x64a5ffdc, // 0x88: daddiu $a1,$a1,-36 + + // JIT re-entry fn addr: + 0x00000000, // 0x8c: lui $t9,reentry + 0x00000000, // 0x90: daddiu $t9,$t9,reentry + 0x00000000, // 0x94: dsll $t9,$t9, + 0x00000000, // 0x98: daddiu $t9,$t9, + 0x00000000, // 0x9c: dsll $t9,$t9, + 0x00000000, // 0xa0: daddiu $t9,$t9, + 0x0320f809, // 0xa4: jalr $t9 + 0x00000000, // 0xa8: nop + 0xdfbf00c8, // 0xac: ld ra, 200(sp) + 0xdfbe00c0, // 0xb0: ld fp, 192(sp) + 0xdfb900b8, // 0xb4: ld t9, 184(sp) + 0xdfb800b0, // 0xb8: ld t8, 176(sp) + 0xdfb700a8, // 0xbc: ld s7, 168(sp) + 0xdfb600a0, // 0xc0: ld s6, 160(sp) + 0xdfb50098, // 0xc4: ld s5, 152(sp) + 0xdfb40090, // 0xc8: ld s4, 144(sp) + 0xdfb30088, // 0xcc: ld s3, 136(sp) + 0xdfb20080, // 0xd0: ld s2, 128(sp) + 0xdfb10078, // 0xd4: ld s1, 120(sp) + 0xdfb00070, // 0xd8: ld s0, 112(sp) + 0xdfaf0068, // 0xdc: ld t3, 104(sp) + 0xdfae0060, // 0xe0: ld t2, 96(sp) + 0xdfad0058, // 0xe4: ld t1, 88(sp) + 0xdfac0050, // 0xe8: ld t0, 80(sp) + 0xdfab0048, // 0xec: ld a7, 72(sp) + 0xdfaa0040, // 0xf0: ld a6, 64(sp) + 0xdfa90038, // 0xf4: ld a5, 56(sp) + 0xdfa80030, // 0xf8: ld a4, 48(sp) + 0xdfa70028, // 0xfc: ld a3, 40(sp) + 0xdfa60020, // 0x100: ld a2, 32(sp) + 0xdfa50018, // 0x104: ld a1, 24(sp) + 0xdfa40010, // 0x108: ld a0, 16(sp) + 0xdfa30008, // 0x10c: ld v1, 8(sp) + 0x67bd00d0, // 0x110: daddiu $sp,$sp,208 + 0x0300f825, // 0x114: move $ra, $t8 + 0x03200008, // 0x118: jr $t9 + 0x0040c825, // 0x11c: move $t9, $v0 + }; + + const unsigned ReentryFnAddrOffset = 0x8c; // JIT re-entry fn addr lui + const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + + uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); + + uint32_t CallMgrLUi = + 0x3c040000 | (((CallMgrAddr + 0x800080008000) >> 48) & 0xFFFF); + uint32_t CallMgrDADDiu = + 0x64840000 | (((CallMgrAddr + 0x80008000) >> 32) & 0xFFFF); + uint32_t CallMgrDSLL = 0x00042438; + uint32_t CallMgrDADDiu2 = + 0x64840000 | ((((CallMgrAddr + 0x8000) >> 16) & 0xFFFF)); + uint32_t CallMgrDSLL2 = 0x00042438; + uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr)&0xFFFF); + + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrDADDiu, + sizeof(CallMgrDADDiu)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 8), &CallMgrDSLL, + sizeof(CallMgrDSLL)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 12), &CallMgrDADDiu2, + sizeof(CallMgrDADDiu2)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 16), &CallMgrDSLL2, + sizeof(CallMgrDSLL2)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 20), &CallMgrDADDiu3, + sizeof(CallMgrDADDiu3)); + + uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); + + uint32_t ReentryLUi = + 0x3c190000 | (((ReentryAddr + 0x800080008000) >> 48) & 0xFFFF); + + uint32_t ReentryDADDiu = + 0x67390000 | (((ReentryAddr + 0x80008000) >> 32) & 0xFFFF); + + uint32_t ReentryDSLL = 0x0019cc38; + + uint32_t ReentryDADDiu2 = + 0x67390000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); + + uint32_t ReentryDSLL2 = 0x0019cc38; + + uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr)&0xFFFF); + + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryDADDiu, + sizeof(ReentryDADDiu)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 8), &ReentryDSLL, + sizeof(ReentryDSLL)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 12), &ReentryDADDiu2, + sizeof(ReentryDADDiu2)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 16), &ReentryDSLL2, + sizeof(ReentryDSLL2)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 20), &ReentryDADDiu3, + sizeof(ReentryDADDiu3)); +} + +void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { + + uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); + + uint64_t HeighestAddr = ((ResolveAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolveAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolveAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra + Trampolines[10 * I + 1] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,resolveAddr + Trampolines[10 * I + 2] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(resolveAddr) + Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 6] = 0x67390000 | (ResolveAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 + Trampolines[10 * I + 8] = 0x00000000; // nop + Trampolines[10 * I + 9] = 0x00000000; // nop + } +} + +Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // stub2: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .dword 0x0 + // ptr2: + // .dword 0x0 + // + // ... + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + unsigned PageSize = sys::Process::getPageSize(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); + uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); + + for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { + uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((PtrAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[8 * I + 0] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,ptr1 + Stub[8 * I + 1] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(ptr) + Stub[8 * I + 2] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 3] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Stub[8 * I + 4] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 5] = 0xdf390000 | (PtrAddr & 0xFFFF); // ld $t9,%lo(ptr) + Stub[8 * I + 6] = 0x03200008; // jr $t9 + Stub[8 * I + 7] = 0x00000000; // nop + } + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp index d6005d24a648..6dea64a6e78f 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -42,89 +42,110 @@ void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledName, void LLVMOrcDisposeMangledSymbol(char *MangledName) { delete[] MangledName; } -LLVMOrcErrorCode -LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - LLVMOrcLazyCompileCallbackFn Callback, - void *CallbackCtx) { +LLVMErrorRef LLVMOrcCreateLazyCompileCallback( + LLVMOrcJITStackRef JITStack, LLVMOrcTargetAddress *RetAddr, + LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.createLazyCompileCallback(*RetAddr, Callback, CallbackCtx); + if (auto Addr = J.createLazyCompileCallback(Callback, CallbackCtx)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); } -LLVMOrcErrorCode LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress InitAddr) { +LLVMErrorRef LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress InitAddr) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.createIndirectStub(StubName, InitAddr); + return wrap(J.createIndirectStub(StubName, InitAddr)); } -LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress NewAddr) { +LLVMErrorRef LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress NewAddr) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.setIndirectStubPointer(StubName, NewAddr); + return wrap(J.setIndirectStubPointer(StubName, NewAddr)); } -LLVMOrcErrorCode -LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { +LLVMErrorRef LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); std::unique_ptr<Module> M(unwrap(Mod)); - return J.addIRModuleEager(*RetHandle, std::move(M), SymbolResolver, - SymbolResolverCtx); + if (auto Handle = + J.addIRModuleEager(std::move(M), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); } -LLVMOrcErrorCode -LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { +LLVMErrorRef LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); std::unique_ptr<Module> M(unwrap(Mod)); - return J.addIRModuleLazy(*RetHandle, std::move(M), SymbolResolver, - SymbolResolverCtx); + if (auto Handle = + J.addIRModuleLazy(std::move(M), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); } -LLVMOrcErrorCode -LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMMemoryBufferRef Obj, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { +LLVMErrorRef LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMMemoryBufferRef Obj, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); std::unique_ptr<MemoryBuffer> O(unwrap(Obj)); - return J.addObject(*RetHandle, std::move(O), SymbolResolver, - SymbolResolverCtx); + if (auto Handle = + J.addObject(std::move(O), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); } -LLVMOrcErrorCode LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle H) { +LLVMErrorRef LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle H) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.removeModule(H); + return wrap(J.removeModule(H)); } -LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - const char *SymbolName) { +LLVMErrorRef LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + const char *SymbolName) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.findSymbolAddress(*RetAddr, SymbolName, true); + if (auto Addr = J.findSymbolAddress(SymbolName, true)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); } -LLVMOrcErrorCode LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - LLVMOrcModuleHandle H, - const char *SymbolName) { +LLVMErrorRef LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + LLVMOrcModuleHandle H, + const char *SymbolName) { OrcCBindingsStack &J = *unwrap(JITStack); - return J.findSymbolAddressIn(*RetAddr, H, SymbolName, true); + if (auto Addr = J.findSymbolAddressIn(H, SymbolName, true)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); } -LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { +LLVMErrorRef LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { auto *J = unwrap(JITStack); auto Err = J->shutdown(); delete J; - return Err; + return wrap(std::move(Err)); } void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index b9f8a370d2f0..817a4b89bfb0 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -77,9 +77,9 @@ public: }; template <> - class GenericLayerImpl<orc::RTDyldObjectLinkingLayer> : public GenericLayer { + class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer { private: - using LayerT = orc::RTDyldObjectLinkingLayer; + using LayerT = orc::LegacyRTDyldObjectLinkingLayer; public: GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} @@ -107,10 +107,10 @@ class OrcCBindingsStack { public: using CompileCallbackMgr = orc::JITCompileCallbackManager; - using ObjLayerT = orc::RTDyldObjectLinkingLayer; - using CompileLayerT = orc::IRCompileLayer<ObjLayerT, orc::SimpleCompiler>; + using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer; + using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>; using CODLayerT = - orc::CompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>; + orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>; using CallbackManagerBuilder = std::function<std::unique_ptr<CompileCallbackMgr>()>; @@ -129,20 +129,21 @@ private: : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), ExternalResolverCtx(std::move(ExternalResolverCtx)) {} - orc::SymbolFlagsMap - lookupFlags(const orc::SymbolNameSet &Symbols) override { - orc::SymbolFlagsMap SymbolFlags; + orc::SymbolNameSet + getResponsibilitySet(const orc::SymbolNameSet &Symbols) override { + orc::SymbolNameSet Result; for (auto &S : Symbols) { - if (auto Sym = findSymbol(*S)) - SymbolFlags[S] = Sym.getFlags(); - else if (auto Err = Sym.takeError()) { + if (auto Sym = findSymbol(*S)) { + if (!Sym.getFlags().isStrong()) + Result.insert(S); + } else if (auto Err = Sym.takeError()) { Stack.reportError(std::move(Err)); - return orc::SymbolFlagsMap(); + return orc::SymbolNameSet(); } } - return SymbolFlags; + return Result; } orc::SymbolNameSet @@ -182,10 +183,17 @@ private: // 2. Runtime overrides. // 3. External resolver (if present). - if (auto Sym = Stack.CODLayer.findSymbol(Name, true)) - return Sym; - else if (auto Err = Sym.takeError()) - return Sym.takeError(); + if (Stack.CODLayer) { + if (auto Sym = Stack.CODLayer->findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } else { + if (auto Sym = Stack.CompileLayer.findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) return Sym; @@ -205,8 +213,8 @@ private: public: OrcCBindingsStack(TargetMachine &TM, IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)), - DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), + : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), + IndirectStubsMgr(IndirectStubsMgrBuilder()), ObjectLayer(ES, [this](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); @@ -226,31 +234,19 @@ public: this->notifyFreed(K, Obj); }), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), - CODLayer(ES, CompileLayer, - [this](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && - "No resolver for module K"); - return ResolverI->second; - }, - [this](orc::VModuleKey K, - std::shared_ptr<orc::SymbolResolver> Resolver) { - assert(!Resolvers.count(K) && "Resolver already present"); - Resolvers[K] = std::move(Resolver); - }, - [](Function &F) { return std::set<Function *>({&F}); }, - *this->CCMgr, std::move(IndirectStubsMgrBuilder), false), + CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), + std::move(IndirectStubsMgrBuilder), Resolvers)), CXXRuntimeOverrides( [this](const std::string &S) { return mangle(S); }) {} - LLVMOrcErrorCode shutdown() { + Error shutdown() { // Run any destructors registered with __cxa_atexit. CXXRuntimeOverrides.runDestructors(); // Run any IR destructors. for (auto &DtorRunner : IRStaticDestructorRunners) if (auto Err = DtorRunner.runViaLayer(*this)) - return mapError(std::move(Err)); - return LLVMOrcErrSuccess; + return Err; + return Error::success(); } std::string mangle(StringRef Name) { @@ -267,35 +263,28 @@ public: return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr)); } - - LLVMOrcErrorCode - createLazyCompileCallback(JITTargetAddress &RetAddr, - LLVMOrcLazyCompileCallbackFn Callback, + Expected<JITTargetAddress> + createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { auto WrappedCallback = [=]() -> JITTargetAddress { return Callback(wrap(this), CallbackCtx); }; - if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) { - RetAddr = *CCAddr; - return LLVMOrcErrSuccess; - } else - return mapError(CCAddr.takeError()); + return CCMgr->getCompileCallback(std::move(WrappedCallback)); } - LLVMOrcErrorCode createIndirectStub(StringRef StubName, - JITTargetAddress Addr) { - return mapError( - IndirectStubsMgr->createStub(StubName, Addr, JITSymbolFlags::Exported)); + Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) { + return IndirectStubsMgr->createStub(StubName, Addr, + JITSymbolFlags::Exported); } - LLVMOrcErrorCode setIndirectStubPointer(StringRef Name, - JITTargetAddress Addr) { - return mapError(IndirectStubsMgr->updatePointer(Name, Addr)); + Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) { + return IndirectStubsMgr->updatePointer(Name, Addr); } + template <typename LayerT> - LLVMOrcErrorCode - addIRModule(orc::VModuleKey &RetKey, LayerT &Layer, std::unique_ptr<Module> M, + Expected<orc::VModuleKey> + addIRModule(LayerT &Layer, std::unique_ptr<Module> M, std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { @@ -313,79 +302,84 @@ public: DtorNames.push_back(mangle(Dtor.Func->getName())); // Add the module to the JIT. - RetKey = ES.allocateVModule(); - Resolvers[RetKey] = std::make_shared<CBindingsResolver>( - *this, ExternalResolver, ExternalResolverCtx); - if (auto Err = Layer.addModule(RetKey, std::move(M))) - return mapError(std::move(Err)); + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver, + ExternalResolverCtx); + if (auto Err = Layer.addModule(K, std::move(M))) + return std::move(Err); - KeyLayers[RetKey] = detail::createGenericLayer(Layer); + KeyLayers[K] = detail::createGenericLayer(Layer); // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), - RetKey); + orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), K); if (auto Err = CtorRunner.runViaLayer(*this)) - return mapError(std::move(Err)); + return std::move(Err); - IRStaticDestructorRunners.emplace_back(std::move(DtorNames), RetKey); + IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K); - return LLVMOrcErrSuccess; + return K; } - LLVMOrcErrorCode addIRModuleEager(orc::VModuleKey &RetKey, - std::unique_ptr<Module> M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return addIRModule(RetKey, CompileLayer, std::move(M), + Expected<orc::VModuleKey> + addIRModuleEager(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(CompileLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode addIRModuleLazy(orc::VModuleKey &RetKey, - std::unique_ptr<Module> M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return addIRModule(RetKey, CODLayer, std::move(M), + Expected<orc::VModuleKey> + addIRModuleLazy(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + if (!CODLayer) + return make_error<StringError>("Can not add lazy module: No compile " + "callback manager available", + inconvertibleErrorCode()); + + return addIRModule(*CODLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode removeModule(orc::VModuleKey K) { + Error removeModule(orc::VModuleKey K) { // FIXME: Should error release the module key? if (auto Err = KeyLayers[K]->removeModule(K)) - return mapError(std::move(Err)); + return Err; ES.releaseVModule(K); KeyLayers.erase(K); - return LLVMOrcErrSuccess; + return Error::success(); } - LLVMOrcErrorCode addObject(orc::VModuleKey &RetKey, - std::unique_ptr<MemoryBuffer> ObjBuffer, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { + Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { if (auto Obj = object::ObjectFile::createObjectFile( ObjBuffer->getMemBufferRef())) { - RetKey = ES.allocateVModule(); - Resolvers[RetKey] = std::make_shared<CBindingsResolver>( + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>( *this, ExternalResolver, ExternalResolverCtx); - if (auto Err = ObjectLayer.addObject(RetKey, std::move(ObjBuffer))) - return mapError(std::move(Err)); + if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer))) + return std::move(Err); - KeyLayers[RetKey] = detail::createGenericLayer(ObjectLayer); + KeyLayers[K] = detail::createGenericLayer(ObjectLayer); - return LLVMOrcErrSuccess; + return K; } else - return mapError(Obj.takeError()); + return Obj.takeError(); } JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) return Sym; - return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); + if (CODLayer) + return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly); + return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); } JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, @@ -394,45 +388,39 @@ public: return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); } - LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr, - const std::string &Name, - bool ExportedSymbolsOnly) { - RetAddr = 0; + Expected<JITTargetAddress> findSymbolAddress(const std::string &Name, + bool ExportedSymbolsOnly) { if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) { // Successful lookup, non-null symbol: - if (auto AddrOrErr = Sym.getAddress()) { - RetAddr = *AddrOrErr; - return LLVMOrcErrSuccess; - } else - return mapError(AddrOrErr.takeError()); + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); } else if (auto Err = Sym.takeError()) { // Lookup failure - report error. - return mapError(std::move(Err)); + return std::move(Err); } - // Otherwise we had a successful lookup but got a null result. We already - // set RetAddr to '0' above, so just return success. - return LLVMOrcErrSuccess; + + // No symbol not found. Return 0. + return 0; } - LLVMOrcErrorCode findSymbolAddressIn(JITTargetAddress &RetAddr, - orc::VModuleKey K, - const std::string &Name, - bool ExportedSymbolsOnly) { - RetAddr = 0; + Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K, + const std::string &Name, + bool ExportedSymbolsOnly) { if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { // Successful lookup, non-null symbol: - if (auto AddrOrErr = Sym.getAddress()) { - RetAddr = *AddrOrErr; - return LLVMOrcErrSuccess; - } else - return mapError(AddrOrErr.takeError()); + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); } else if (auto Err = Sym.takeError()) { // Lookup failure - report error. - return mapError(std::move(Err)); + return std::move(Err); } - // Otherwise we had a successful lookup but got a null result. We already - // set RetAddr to '0' above, so just return success. - return LLVMOrcErrSuccess; + + // Symbol not found. Return 0. + return 0; } const std::string &getErrorMessage() const { return ErrMsg; } @@ -455,17 +443,45 @@ public: } private: + using ResolverMap = + std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>; + + static std::unique_ptr<CompileCallbackMgr> + createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) { + auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0); + if (!CCMgr) { + // FIXME: It would be good if we could report this somewhere, but we do + // have an instance yet. + logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: "); + return nullptr; + } + return std::move(*CCMgr); + } - LLVMOrcErrorCode mapError(Error Err) { - LLVMOrcErrorCode Result = LLVMOrcErrSuccess; - handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { - // Handler of last resort. - Result = LLVMOrcErrGeneric; - ErrMsg = ""; - raw_string_ostream ErrStream(ErrMsg); - EIB.log(ErrStream); - }); - return Result; + static std::unique_ptr<CODLayerT> + createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer, + CompileCallbackMgr *CCMgr, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder, + ResolverMap &Resolvers) { + // If there is no compile callback manager available we can not create a + // compile on demand layer. + if (!CCMgr) + return nullptr; + + return llvm::make_unique<CODLayerT>( + ES, CompileLayer, + [&Resolvers](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && "No resolver for module K"); + return ResolverI->second; + }, + [&Resolvers](orc::VModuleKey K, + std::shared_ptr<orc::SymbolResolver> Resolver) { + assert(!Resolvers.count(K) && "Resolver already present"); + Resolvers[K] = std::move(Resolver); + }, + [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr, + std::move(IndirectStubsMgrBuilder), false); } void reportError(Error Err) { @@ -476,13 +492,17 @@ private: void notifyFinalized(orc::VModuleKey K, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); for (auto &Listener : EventListeners) - Listener->NotifyObjectEmitted(Obj, LoadedObjInfo); + Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo); } void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); for (auto &Listener : EventListeners) - Listener->NotifyFreeingObject(Obj); + Listener->notifyFreeingObject(Key); } orc::ExecutionSession ES; @@ -497,15 +517,15 @@ private: ObjLayerT ObjectLayer; CompileLayerT CompileLayer; - CODLayerT CODLayer; + std::unique_ptr<CODLayerT> CODLayer; std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; - orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; - std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; + orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides; + std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; std::string ErrMsg; - std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers; + ResolverMap Resolvers; }; } // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index 4def579e7097..617bc2fc64b5 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -128,7 +128,7 @@ void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; for (auto &KV : CtorDtorsMap) - cantFail(CtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) + cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) .runViaLayer(LazyEmitLayer)); CtorDtorsMap.clear(); diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index abe89ce70af9..36e7e83a8bab 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -144,26 +144,29 @@ class OrcMCJITReplacement : public ExecutionEngine { public: LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} - SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) override { - SymbolFlagsMap SymbolFlags; + SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override { + SymbolNameSet Result; for (auto &S : Symbols) { if (auto Sym = M.findMangledSymbol(*S)) { - SymbolFlags[S] = Sym.getFlags(); + if (!Sym.getFlags().isStrong()) + Result.insert(S); } else if (auto Err = Sym.takeError()) { M.reportError(std::move(Err)); - return SymbolFlagsMap(); + return SymbolNameSet(); } else { if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) { - SymbolFlags[S] = Sym2.getFlags(); + if (!Sym2.getFlags().isStrong()) + Result.insert(S); } else if (auto Err = Sym2.takeError()) { M.reportError(std::move(Err)); - return SymbolFlagsMap(); - } + return SymbolNameSet(); + } else + Result.insert(S); } } - return SymbolFlags; + return Result; } SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, @@ -272,14 +275,14 @@ public: { unsigned CtorId = 0, DtorId = 0; for (auto Ctor : orc::getConstructors(*M)) { - std::string NewCtorName = ("$static_ctor." + Twine(CtorId++)).str(); + std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str(); Ctor.Func->setName(NewCtorName); Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); CtorNames.push_back(mangle(NewCtorName)); } for (auto Dtor : orc::getDestructors(*M)) { - std::string NewDtorName = ("$static_dtor." + Twine(DtorId++)).str(); + std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str(); dbgs() << "Found dtor: " << NewDtorName << "\n"; Dtor.Func->setName(NewDtorName); Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); @@ -458,8 +461,8 @@ private: return MangledName; } - using ObjectLayerT = RTDyldObjectLinkingLayer; - using CompileLayerT = IRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; + using ObjectLayerT = LegacyRTDyldObjectLinkingLayer; + using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; ExecutionSession ES; diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 71b4b73ca6d3..299d76183cd4 100644 --- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -14,57 +14,56 @@ namespace { using namespace llvm; using namespace llvm::orc; -class VSOSearchOrderResolver : public JITSymbolResolver { +class JITDylibSearchOrderResolver : public JITSymbolResolver { public: - VSOSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} + JITDylibSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} - Expected<LookupResult> lookup(const LookupSet &Symbols) { - auto &ES = MR.getTargetVSO().getExecutionSession(); + void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); SymbolNameSet InternedSymbols; + // Intern the requested symbols: lookup takes interned strings. for (auto &S : Symbols) - InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); - + InternedSymbols.insert(ES.intern(S)); + + // Build an OnResolve callback to unwrap the interned strings and pass them + // to the OnResolved callback. + // FIXME: Switch to move capture of OnResolved once we have c++14. + auto OnResolvedWithUnwrap = + [OnResolved](Expected<SymbolMap> InternedResult) { + if (!InternedResult) { + OnResolved(InternedResult.takeError()); + return; + } + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + OnResolved(Result); + }; + + // We're not waiting for symbols to be ready. Just log any errors. + auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; + + // Register dependencies for all symbols contained in this set. auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { MR.addDependenciesForAll(Deps); }; - auto InternedResult = - MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { - return ES.lookup(VSOs, InternedSymbols, RegisterDependencies, false); - }); - - if (!InternedResult) - return InternedResult.takeError(); - - LookupResult Result; - for (auto &KV : *InternedResult) - Result[*KV.first] = std::move(KV.second); - - return Result; + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + ES.lookup(SearchOrder, InternedSymbols, OnResolvedWithUnwrap, OnReady, + RegisterDependencies); } - Expected<LookupFlagsResult> lookupFlags(const LookupSet &Symbols) { - auto &ES = MR.getTargetVSO().getExecutionSession(); - - SymbolNameSet InternedSymbols; - - for (auto &S : Symbols) - InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); - - SymbolFlagsMap InternedResult; - MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { - // An empty search order is pathalogical, but allowed. - if (VSOs.empty()) - return; + Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { + LookupSet Result; - assert(VSOs.front() && "VSOList entry can not be null"); - InternedResult = VSOs.front()->lookupFlags(InternedSymbols); - }); - - LookupFlagsResult Result; - for (auto &KV : InternedResult) - Result[*KV.first] = std::move(KV.second); + for (auto &KV : MR.getSymbols()) { + if (Symbols.count(*KV.first)) + Result.insert(*KV.first); + } return Result; } @@ -78,52 +77,41 @@ private: namespace llvm { namespace orc { -RTDyldObjectLinkingLayer2::RTDyldObjectLinkingLayer2( +RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, - NotifyLoadedFunction NotifyLoaded, NotifyFinalizedFunction NotifyFinalized) + NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted) : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), NotifyLoaded(std::move(NotifyLoaded)), - NotifyFinalized(std::move(NotifyFinalized)), ProcessAllSections(false) {} + NotifyEmitted(std::move(NotifyEmitted)) {} -void RTDyldObjectLinkingLayer2::emit(MaterializationResponsibility R, - VModuleKey K, - std::unique_ptr<MemoryBuffer> O) { +void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); - auto &ES = getExecutionSession(); - - auto ObjFile = object::ObjectFile::createObjectFile(*O); - if (!ObjFile) { - getExecutionSession().reportError(ObjFile.takeError()); - R.failMaterialization(); - } - - auto MemoryManager = GetMemoryManager(K); - - VSOSearchOrderResolver Resolver(R); - auto RTDyld = llvm::make_unique<RuntimeDyld>(*MemoryManager, Resolver); - RTDyld->setProcessAllSections(ProcessAllSections); + // This method launches an asynchronous link step that will fulfill our + // materialization responsibility. We need to switch R to be heap + // allocated before that happens so it can live as long as the asynchronous + // link needs it to (i.e. it must be able to outlive this method). + auto SharedR = std::make_shared<MaterializationResponsibility>(std::move(R)); - { - std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto &ES = getExecutionSession(); - assert(!ActiveRTDylds.count(K) && - "An active RTDyld already exists for this key?"); - ActiveRTDylds[K] = RTDyld.get(); + auto Obj = object::ObjectFile::createObjectFile(*O); - assert(!MemMgrs.count(K) && - "A memory manager already exists for this key?"); - MemMgrs[K] = std::move(MemoryManager); + if (!Obj) { + getExecutionSession().reportError(Obj.takeError()); + SharedR->failMaterialization(); + return; } - auto Info = RTDyld->loadObject(**ObjFile); - + // Collect the internal symbols from the object file: We will need to + // filter these later. + auto InternalSymbols = std::make_shared<std::set<StringRef>>(); { - std::set<StringRef> InternalSymbols; - for (auto &Sym : (*ObjFile)->symbols()) { + for (auto &Sym : (*Obj)->symbols()) { if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) { if (auto SymName = Sym.getName()) - InternalSymbols.insert(*SymName); + InternalSymbols->insert(*SymName); else { ES.reportError(SymName.takeError()); R.failMaterialization(); @@ -131,46 +119,97 @@ void RTDyldObjectLinkingLayer2::emit(MaterializationResponsibility R, } } } - - SymbolMap Symbols; - for (auto &KV : RTDyld->getSymbolTable()) - if (!InternalSymbols.count(KV.first)) - Symbols[ES.getSymbolStringPool().intern(KV.first)] = KV.second; - - R.resolve(Symbols); } - if (NotifyLoaded) - NotifyLoaded(K, **ObjFile, *Info); - - RTDyld->finalizeWithMemoryManagerLocking(); + auto K = R.getVModuleKey(); + RuntimeDyld::MemoryManager *MemMgr = nullptr; + // Create a record a memory manager for this object. { + auto Tmp = GetMemoryManager(); std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - ActiveRTDylds.erase(K); + MemMgrs.push_back(std::move(Tmp)); + MemMgr = MemMgrs.back().get(); } - if (RTDyld->hasError()) { - ES.reportError(make_error<StringError>(RTDyld->getErrorString(), - inconvertibleErrorCode())); - R.failMaterialization(); - return; + JITDylibSearchOrderResolver Resolver(*SharedR); + + /* Thoughts on proper cross-dylib weak symbol handling: + * + * Change selection of canonical defs to be a manually triggered process, and + * add a 'canonical' bit to symbol definitions. When canonical def selection + * is triggered, sweep the JITDylibs to mark defs as canonical, discard + * duplicate defs. + */ + jitLinkForORC( + **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, + [this, K, SharedR, &Obj, InternalSymbols]( + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) { + return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), + ResolvedSymbols, *InternalSymbols); + }, + [this, K, SharedR](Error Err) { + onObjEmit(K, *SharedR, std::move(Err)); + }); +} + +Error RTDyldObjectLinkingLayer::onObjLoad( + VModuleKey K, MaterializationResponsibility &R, object::ObjectFile &Obj, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> Resolved, + std::set<StringRef> &InternalSymbols) { + SymbolFlagsMap ExtraSymbolsToClaim; + SymbolMap Symbols; + for (auto &KV : Resolved) { + // Scan the symbols and add them to the Symbols map for resolution. + + // We never claim internal symbols. + if (InternalSymbols.count(KV.first)) + continue; + + auto InternedName = getExecutionSession().intern(KV.first); + auto Flags = KV.second.getFlags(); + + // Override object flags and claim responsibility for symbols if + // requested. + if (OverrideObjectFlags || AutoClaimObjectSymbols) { + auto I = R.getSymbols().find(InternedName); + + if (OverrideObjectFlags && I != R.getSymbols().end()) + Flags = JITSymbolFlags::stripTransientFlags(I->second); + else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) + ExtraSymbolsToClaim[InternedName] = Flags; + } + + Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); } - R.finalize(); + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) + return Err; + + R.resolve(Symbols); + + if (NotifyLoaded) + NotifyLoaded(K, Obj, *LoadedObjInfo); - if (NotifyFinalized) - NotifyFinalized(K); + return Error::success(); } -void RTDyldObjectLinkingLayer2::mapSectionAddress( - VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) const { - std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - auto ActiveRTDyldItr = ActiveRTDylds.find(K); +void RTDyldObjectLinkingLayer::onObjEmit(VModuleKey K, + MaterializationResponsibility &R, + Error Err) { + if (Err) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + R.emit(); - assert(ActiveRTDyldItr != ActiveRTDylds.end() && - "No active RTDyld instance found for key"); - ActiveRTDyldItr->second->mapSectionAddress(LocalAddress, TargetAddr); + if (NotifyEmitted) + NotifyEmitted(K); } } // End namespace orc. diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp new file mode 100644 index 000000000000..9525b168fbd3 --- /dev/null +++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -0,0 +1,65 @@ +//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities +//h-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Transforms/Utils/Cloning.h" + +namespace llvm { +namespace orc { + +ThreadSafeModule cloneToNewContext(ThreadSafeModule &TSM, + GVPredicate ShouldCloneDef, + GVModifier UpdateClonedDefSource) { + assert(TSM && "Can not clone null module"); + + if (!ShouldCloneDef) + ShouldCloneDef = [](const GlobalValue &) { return true; }; + + auto Lock = TSM.getContextLock(); + + SmallVector<char, 1> ClonedModuleBuffer; + + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(*TSM.getModule(), VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDef(*GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); + + if (UpdateClonedDefSource) + for (auto *GV : ClonedDefsInSrc) + UpdateClonedDefSource(*GV); + + BitcodeWriter BCWriter(ClonedModuleBuffer); + + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } + + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + ThreadSafeContext NewTSCtx(llvm::make_unique<LLVMContext>()); + + auto ClonedModule = + cantFail(parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext())); + ClonedModule->setModuleIdentifier(TSM.getModule()->getName()); + return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx)); +} + +} // end namespace orc +} // end namespace llvm |