summaryrefslogtreecommitdiff
path: root/lib/ExecutionEngine/Orc
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2018-07-28 10:51:19 +0000
committerDimitry Andric <dim@FreeBSD.org>2018-07-28 10:51:19 +0000
commiteb11fae6d08f479c0799db45860a98af528fa6e7 (patch)
tree44d492a50c8c1a7eb8e2d17ea3360ec4d066f042 /lib/ExecutionEngine/Orc
parentb8a2042aa938069e862750553db0e4d82d25822c (diff)
Notes
Diffstat (limited to 'lib/ExecutionEngine/Orc')
-rw-r--r--lib/ExecutionEngine/Orc/CMakeLists.txt15
-rw-r--r--lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp343
-rw-r--r--lib/ExecutionEngine/Orc/Core.cpp1690
-rw-r--r--lib/ExecutionEngine/Orc/ExecutionUtils.cpp154
-rw-r--r--lib/ExecutionEngine/Orc/IRCompileLayer.cpp44
-rw-r--r--lib/ExecutionEngine/Orc/IRTransformLayer.cpp34
-rw-r--r--lib/ExecutionEngine/Orc/IndirectionUtils.cpp107
-rw-r--r--lib/ExecutionEngine/Orc/LLJIT.cpp134
-rw-r--r--lib/ExecutionEngine/Orc/LLVMBuild.txt4
-rw-r--r--lib/ExecutionEngine/Orc/Layer.cpp106
-rw-r--r--lib/ExecutionEngine/Orc/Legacy.cpp68
-rw-r--r--lib/ExecutionEngine/Orc/NullResolver.cpp16
-rw-r--r--lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp34
-rw-r--r--lib/ExecutionEngine/Orc/OrcCBindings.cpp48
-rw-r--r--lib/ExecutionEngine/Orc/OrcCBindingsStack.h382
-rw-r--r--lib/ExecutionEngine/Orc/OrcError.cpp25
-rw-r--r--lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp9
-rw-r--r--lib/ExecutionEngine/Orc/OrcMCJITReplacement.h224
-rw-r--r--lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp177
19 files changed, 3364 insertions, 250 deletions
diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt
index f83e002c758f..a7500ef20f37 100644
--- a/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -1,12 +1,21 @@
add_llvm_library(LLVMOrcJIT
+ CompileOnDemandLayer.cpp
+ Core.cpp
ExecutionUtils.cpp
IndirectionUtils.cpp
+ IRCompileLayer.cpp
+ IRTransformLayer.cpp
+ Legacy.cpp
+ Layer.cpp
+ LLJIT.cpp
NullResolver.cpp
+ ObjectTransformLayer.cpp
OrcABISupport.cpp
OrcCBindings.cpp
OrcError.cpp
OrcMCJITReplacement.cpp
RPCUtils.cpp
+ RTDyldObjectLinkingLayer.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
@@ -14,3 +23,9 @@ add_llvm_library(LLVMOrcJIT
DEPENDS
intrinsics_gen
)
+
+target_link_libraries(LLVMOrcJIT
+ PRIVATE
+ LLVMBitReader
+ LLVMBitWriter
+ )
diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
new file mode 100644
index 000000000000..d42e7b05ba67
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -0,0 +1,343 @@
+//===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/IR/Mangler.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+namespace {
+
+template <typename MaterializerFtor>
+class LambdaValueMaterializer final : public ValueMaterializer {
+public:
+ LambdaValueMaterializer(MaterializerFtor M) : M(std::move(M)) {}
+
+ Value *materialize(Value *V) final { return M(V); }
+
+private:
+ MaterializerFtor M;
+};
+
+template <typename MaterializerFtor>
+LambdaValueMaterializer<MaterializerFtor>
+createLambdaValueMaterializer(MaterializerFtor M) {
+ return LambdaValueMaterializer<MaterializerFtor>(std::move(M));
+}
+} // namespace
+
+static void extractAliases(MaterializationResponsibility &R, Module &M,
+ MangleAndInterner &Mangle) {
+ SymbolAliasMap Aliases;
+
+ std::vector<GlobalAlias *> ModAliases;
+ for (auto &A : M.aliases())
+ ModAliases.push_back(&A);
+
+ for (auto *A : ModAliases) {
+ Constant *Aliasee = A->getAliasee();
+ assert(A->hasName() && "Anonymous alias?");
+ assert(Aliasee->hasName() && "Anonymous aliasee");
+ std::string AliasName = A->getName();
+
+ Aliases[Mangle(AliasName)] = SymbolAliasMapEntry(
+ {Mangle(Aliasee->getName()), JITSymbolFlags::fromGlobalValue(*A)});
+
+ if (isa<Function>(Aliasee)) {
+ auto *F = cloneFunctionDecl(M, *cast<Function>(Aliasee));
+ A->replaceAllUsesWith(F);
+ A->eraseFromParent();
+ F->setName(AliasName);
+ } else if (isa<GlobalValue>(Aliasee)) {
+ auto *G = cloneGlobalVariableDecl(M, *cast<GlobalVariable>(Aliasee));
+ A->replaceAllUsesWith(G);
+ A->eraseFromParent();
+ G->setName(AliasName);
+ }
+ }
+
+ R.replace(symbolAliases(std::move(Aliases)));
+}
+
+static std::unique_ptr<Module>
+extractAndClone(Module &M, LLVMContext &NewContext, StringRef Suffix,
+ function_ref<bool(const GlobalValue *)> ShouldCloneDefinition) {
+ SmallVector<char, 1> ClonedModuleBuffer;
+
+ {
+ std::set<GlobalValue *> ClonedDefsInSrc;
+ ValueToValueMapTy VMap;
+ auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
+ if (ShouldCloneDefinition(GV)) {
+ ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
+ return true;
+ }
+ return false;
+ });
+
+ for (auto *GV : ClonedDefsInSrc) {
+ // Delete the definition and bump the linkage in the source module.
+ if (isa<Function>(GV)) {
+ auto &F = *cast<Function>(GV);
+ F.deleteBody();
+ F.setPersonalityFn(nullptr);
+ } else if (isa<GlobalVariable>(GV)) {
+ cast<GlobalVariable>(GV)->setInitializer(nullptr);
+ } else
+ llvm_unreachable("Unsupported global type");
+
+ GV->setLinkage(GlobalValue::ExternalLinkage);
+ }
+
+ BitcodeWriter BCWriter(ClonedModuleBuffer);
+
+ BCWriter.writeModule(*Tmp);
+ BCWriter.writeSymtab();
+ BCWriter.writeStrtab();
+ }
+
+ MemoryBufferRef ClonedModuleBufferRef(
+ StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
+ "cloned module buffer");
+
+ auto ClonedModule =
+ cantFail(parseBitcodeFile(ClonedModuleBufferRef, NewContext));
+ ClonedModule->setModuleIdentifier((M.getName() + Suffix).str());
+ return ClonedModule;
+}
+
+static std::unique_ptr<Module> extractGlobals(Module &M,
+ LLVMContext &NewContext) {
+ return extractAndClone(M, NewContext, ".globals", [](const GlobalValue *GV) {
+ return isa<GlobalVariable>(GV);
+ });
+}
+
+namespace llvm {
+namespace orc {
+
+class ExtractingIRMaterializationUnit : public IRMaterializationUnit {
+public:
+ ExtractingIRMaterializationUnit(ExecutionSession &ES,
+ CompileOnDemandLayer2 &Parent,
+ std::unique_ptr<Module> M)
+ : IRMaterializationUnit(ES, std::move(M)), Parent(Parent) {}
+
+ ExtractingIRMaterializationUnit(std::unique_ptr<Module> M,
+ SymbolFlagsMap SymbolFlags,
+ SymbolNameToDefinitionMap SymbolToDefinition,
+ CompileOnDemandLayer2 &Parent)
+ : IRMaterializationUnit(std::move(M), std::move(SymbolFlags),
+ std::move(SymbolToDefinition)),
+ Parent(Parent) {}
+
+private:
+ void materialize(MaterializationResponsibility R) override {
+ // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the
+ // extracted module key, extracted module, and source module key
+ // together. This could be used, for example, to provide a specific
+ // memory manager instance to the linking layer.
+
+ auto RequestedSymbols = R.getRequestedSymbols();
+
+ // Extract the requested functions into a new module.
+ std::unique_ptr<Module> ExtractedFunctionsModule;
+ if (!RequestedSymbols.empty()) {
+ std::string Suffix;
+ std::set<const GlobalValue *> FunctionsToClone;
+ for (auto &Name : RequestedSymbols) {
+ auto I = SymbolToDefinition.find(Name);
+ assert(I != SymbolToDefinition.end() && I->second != nullptr &&
+ "Should have a non-null definition");
+ FunctionsToClone.insert(I->second);
+ Suffix += ".";
+ Suffix += *Name;
+ }
+
+ std::lock_guard<std::mutex> Lock(SourceModuleMutex);
+ ExtractedFunctionsModule =
+ extractAndClone(*M, Parent.GetAvailableContext(), Suffix,
+ [&](const GlobalValue *GV) -> bool {
+ return FunctionsToClone.count(GV);
+ });
+ }
+
+ // Build a new ExtractingIRMaterializationUnit to delegate the unrequested
+ // symbols to.
+ SymbolFlagsMap DelegatedSymbolFlags;
+ IRMaterializationUnit::SymbolNameToDefinitionMap
+ DelegatedSymbolToDefinition;
+ for (auto &KV : SymbolToDefinition) {
+ if (RequestedSymbols.count(KV.first))
+ continue;
+ DelegatedSymbolFlags[KV.first] =
+ JITSymbolFlags::fromGlobalValue(*KV.second);
+ DelegatedSymbolToDefinition[KV.first] = KV.second;
+ }
+
+ if (!DelegatedSymbolFlags.empty()) {
+ assert(DelegatedSymbolFlags.size() ==
+ DelegatedSymbolToDefinition.size() &&
+ "SymbolFlags and SymbolToDefinition should have the same number "
+ "of entries");
+ R.replace(llvm::make_unique<ExtractingIRMaterializationUnit>(
+ std::move(M), std::move(DelegatedSymbolFlags),
+ std::move(DelegatedSymbolToDefinition), Parent));
+ }
+
+ if (ExtractedFunctionsModule)
+ Parent.emitExtractedFunctionsModule(std::move(R),
+ std::move(ExtractedFunctionsModule));
+ }
+
+ void discard(const VSO &V, SymbolStringPtr Name) override {
+ // All original symbols were materialized by the CODLayer and should be
+ // final. The function bodies provided by M should never be overridden.
+ llvm_unreachable("Discard should never be called on an "
+ "ExtractingIRMaterializationUnit");
+ }
+
+ mutable std::mutex SourceModuleMutex;
+ CompileOnDemandLayer2 &Parent;
+};
+
+CompileOnDemandLayer2::CompileOnDemandLayer2(
+ ExecutionSession &ES, IRLayer &BaseLayer, JITCompileCallbackManager &CCMgr,
+ IndirectStubsManagerBuilder BuildIndirectStubsManager,
+ GetAvailableContextFunction GetAvailableContext)
+ : IRLayer(ES), BaseLayer(BaseLayer), CCMgr(CCMgr),
+ BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)),
+ GetAvailableContext(std::move(GetAvailableContext)) {}
+
+Error CompileOnDemandLayer2::add(VSO &V, VModuleKey K,
+ std::unique_ptr<Module> M) {
+ return IRLayer::add(V, K, std::move(M));
+}
+
+void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K,
+ std::unique_ptr<Module> M) {
+ auto &ES = getExecutionSession();
+ assert(M && "M should not be null");
+
+ for (auto &GV : M->global_values())
+ if (GV.hasWeakLinkage())
+ GV.setLinkage(GlobalValue::ExternalLinkage);
+
+ MangleAndInterner Mangle(ES, M->getDataLayout());
+
+ extractAliases(R, *M, Mangle);
+
+ auto GlobalsModule = extractGlobals(*M, GetAvailableContext());
+
+ // Delete the bodies of any available externally functions, rename the
+ // rest, and build the compile callbacks.
+ std::map<SymbolStringPtr, std::pair<JITTargetAddress, JITSymbolFlags>>
+ StubCallbacksAndLinkages;
+ auto &TargetVSO = R.getTargetVSO();
+
+ for (auto &F : M->functions()) {
+ if (F.isDeclaration())
+ continue;
+
+ if (F.hasAvailableExternallyLinkage()) {
+ F.deleteBody();
+ F.setPersonalityFn(nullptr);
+ continue;
+ }
+
+ assert(F.hasName() && "Function should have a name");
+ std::string StubUnmangledName = F.getName();
+ F.setName(F.getName() + "$body");
+ auto StubDecl = cloneFunctionDecl(*M, F);
+ StubDecl->setName(StubUnmangledName);
+ StubDecl->setPersonalityFn(nullptr);
+ StubDecl->setLinkage(GlobalValue::ExternalLinkage);
+ F.replaceAllUsesWith(StubDecl);
+
+ auto StubName = Mangle(StubUnmangledName);
+ auto BodyName = Mangle(F.getName());
+ if (auto CallbackAddr = CCMgr.getCompileCallback(
+ [BodyName, &TargetVSO, &ES]() -> JITTargetAddress {
+ if (auto Sym = lookup({&TargetVSO}, BodyName))
+ return Sym->getAddress();
+ else {
+ ES.reportError(Sym.takeError());
+ return 0;
+ }
+ })) {
+ auto Flags = JITSymbolFlags::fromGlobalValue(F);
+ Flags &= ~JITSymbolFlags::Weak;
+ StubCallbacksAndLinkages[std::move(StubName)] =
+ std::make_pair(*CallbackAddr, Flags);
+ } else {
+ ES.reportError(CallbackAddr.takeError());
+ R.failMaterialization();
+ return;
+ }
+ }
+
+ // Build the stub inits map.
+ IndirectStubsManager::StubInitsMap StubInits;
+ for (auto &KV : StubCallbacksAndLinkages)
+ StubInits[*KV.first] = KV.second;
+
+ // Build the function-body-extracting materialization unit.
+ if (auto Err = R.getTargetVSO().define(
+ llvm::make_unique<ExtractingIRMaterializationUnit>(ES, *this,
+ std::move(M)))) {
+ ES.reportError(std::move(Err));
+ R.failMaterialization();
+ return;
+ }
+
+ // Build the stubs.
+ // FIXME: Remove function bodies materialization unit if stub creation fails.
+ auto &StubsMgr = getStubsManager(TargetVSO);
+ if (auto Err = StubsMgr.createStubs(StubInits)) {
+ ES.reportError(std::move(Err));
+ R.failMaterialization();
+ return;
+ }
+
+ // Resolve and finalize stubs.
+ SymbolMap ResolvedStubs;
+ for (auto &KV : StubCallbacksAndLinkages) {
+ if (auto Sym = StubsMgr.findStub(*KV.first, false))
+ ResolvedStubs[KV.first] = Sym;
+ else
+ llvm_unreachable("Stub went missing");
+ }
+
+ R.resolve(ResolvedStubs);
+
+ BaseLayer.emit(std::move(R), std::move(K), std::move(GlobalsModule));
+}
+
+IndirectStubsManager &CompileOnDemandLayer2::getStubsManager(const VSO &V) {
+ std::lock_guard<std::mutex> Lock(CODLayerMutex);
+ StubManagersMap::iterator I = StubsMgrs.find(&V);
+ if (I == StubsMgrs.end())
+ I = StubsMgrs.insert(std::make_pair(&V, BuildIndirectStubsManager())).first;
+ return *I->second;
+}
+
+void CompileOnDemandLayer2::emitExtractedFunctionsModule(
+ MaterializationResponsibility R, std::unique_ptr<Module> M) {
+ auto K = getExecutionSession().allocateVModule();
+ BaseLayer.emit(std::move(R), std::move(K), std::move(M));
+}
+
+} // end namespace orc
+} // end namespace llvm
diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp
new file mode 100644
index 000000000000..4325d57f73d0
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/Core.cpp
@@ -0,0 +1,1690 @@
+//===----- Core.cpp - Core ORC APIs (MaterializationUnit, VSO, etc.) ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/ExecutionEngine/Orc/OrcError.h"
+#include "llvm/IR/Mangler.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+
+#if LLVM_ENABLE_THREADS
+#include <future>
+#endif
+
+namespace llvm {
+namespace orc {
+
+char FailedToMaterialize::ID = 0;
+char SymbolsNotFound::ID = 0;
+
+RegisterDependenciesFunction NoDependenciesToRegister =
+ RegisterDependenciesFunction();
+
+void MaterializationUnit::anchor() {}
+
+raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) {
+ if (Flags.isWeak())
+ OS << 'W';
+ else if (Flags.isCommon())
+ OS << 'C';
+ else
+ OS << 'S';
+
+ if (Flags.isExported())
+ OS << 'E';
+ else
+ OS << 'H';
+
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) {
+ OS << format("0x%016x", Sym.getAddress()) << " " << Sym.getFlags();
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) {
+ OS << "\"" << *KV.first << "\": " << KV.second;
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) {
+ OS << "{";
+ if (!Symbols.empty()) {
+ OS << " \"" << **Symbols.begin() << "\"";
+ for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end()))
+ OS << ", \"" << *Sym << "\"";
+ }
+ OS << " }";
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) {
+ OS << "{";
+ if (!Symbols.empty()) {
+ OS << " {" << *Symbols.begin() << "}";
+ for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end()))
+ OS << ", {" << Sym << "}";
+ }
+ OS << " }";
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) {
+ OS << "{";
+ if (!SymbolFlags.empty()) {
+ OS << " {\"" << *SymbolFlags.begin()->first
+ << "\": " << SymbolFlags.begin()->second << "}";
+ for (auto &KV :
+ make_range(std::next(SymbolFlags.begin()), SymbolFlags.end()))
+ OS << ", {\"" << *KV.first << "\": " << KV.second << "}";
+ }
+ OS << " }";
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) {
+ OS << "{";
+ if (!Deps.empty()) {
+ OS << " { " << Deps.begin()->first->getName() << ": "
+ << Deps.begin()->second << " }";
+ for (auto &KV : make_range(std::next(Deps.begin()), Deps.end()))
+ OS << ", { " << KV.first->getName() << ": " << KV.second << " }";
+ }
+ OS << " }";
+ return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const VSOList &VSOs) {
+ OS << "[";
+ if (!VSOs.empty()) {
+ assert(VSOs.front() && "VSOList entries must not be null");
+ OS << " " << VSOs.front()->getName();
+ for (auto *V : make_range(std::next(VSOs.begin()), VSOs.end())) {
+ assert(V && "VSOList entries must not be null");
+ OS << ", " << V->getName();
+ }
+ }
+ OS << " ]";
+ return OS;
+}
+
+FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols)
+ : Symbols(std::move(Symbols)) {
+ assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
+}
+
+std::error_code FailedToMaterialize::convertToErrorCode() const {
+ return orcError(OrcErrorCode::UnknownORCError);
+}
+
+void FailedToMaterialize::log(raw_ostream &OS) const {
+ OS << "Failed to materialize symbols: " << Symbols;
+}
+
+SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols)
+ : Symbols(std::move(Symbols)) {
+ assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
+}
+
+std::error_code SymbolsNotFound::convertToErrorCode() const {
+ return orcError(OrcErrorCode::UnknownORCError);
+}
+
+void SymbolsNotFound::log(raw_ostream &OS) const {
+ OS << "Symbols not found: " << Symbols;
+}
+
+void ExecutionSessionBase::legacyFailQuery(AsynchronousSymbolQuery &Q,
+ Error Err) {
+ assert(!!Err && "Error should be in failure state");
+
+ bool SendErrorToQuery;
+ runSessionLocked([&]() {
+ Q.detach();
+ SendErrorToQuery = Q.canStillFail();
+ });
+
+ if (SendErrorToQuery)
+ Q.handleFailed(std::move(Err));
+ else
+ reportError(std::move(Err));
+}
+
+Expected<SymbolMap> ExecutionSessionBase::legacyLookup(
+ ExecutionSessionBase &ES, LegacyAsyncLookupFunction AsyncLookup,
+ SymbolNameSet Names, bool WaitUntilReady,
+ RegisterDependenciesFunction RegisterDependencies) {
+#if LLVM_ENABLE_THREADS
+ // In the threaded case we use promises to return the results.
+ std::promise<SymbolMap> PromisedResult;
+ std::mutex ErrMutex;
+ Error ResolutionError = Error::success();
+ std::promise<void> PromisedReady;
+ Error ReadyError = Error::success();
+ auto OnResolve = [&](Expected<SymbolMap> R) {
+ if (R)
+ PromisedResult.set_value(std::move(*R));
+ else {
+ {
+ ErrorAsOutParameter _(&ResolutionError);
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ ResolutionError = R.takeError();
+ }
+ PromisedResult.set_value(SymbolMap());
+ }
+ };
+
+ std::function<void(Error)> OnReady;
+ if (WaitUntilReady) {
+ OnReady = [&](Error Err) {
+ if (Err) {
+ ErrorAsOutParameter _(&ReadyError);
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ ReadyError = std::move(Err);
+ }
+ PromisedReady.set_value();
+ };
+ } else {
+ OnReady = [&](Error Err) {
+ if (Err)
+ ES.reportError(std::move(Err));
+ };
+ }
+
+#else
+ SymbolMap Result;
+ Error ResolutionError = Error::success();
+ Error ReadyError = Error::success();
+
+ auto OnResolve = [&](Expected<SymbolMap> R) {
+ ErrorAsOutParameter _(&ResolutionError);
+ if (R)
+ Result = std::move(*R);
+ else
+ ResolutionError = R.takeError();
+ };
+
+ std::function<void(Error)> OnReady;
+ if (WaitUntilReady) {
+ OnReady = [&](Error Err) {
+ ErrorAsOutParameter _(&ReadyError);
+ if (Err)
+ ReadyError = std::move(Err);
+ };
+ } else {
+ OnReady = [&](Error Err) {
+ if (Err)
+ ES.reportError(std::move(Err));
+ };
+ }
+#endif
+
+ auto Query = std::make_shared<AsynchronousSymbolQuery>(
+ Names, std::move(OnResolve), std::move(OnReady));
+ // FIXME: This should be run session locked along with the registration code
+ // and error reporting below.
+ SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names));
+
+ // If the query was lodged successfully then register the dependencies,
+ // otherwise fail it with an error.
+ if (UnresolvedSymbols.empty())
+ RegisterDependencies(Query->QueryRegistrations);
+ else {
+ bool DeliverError = runSessionLocked([&]() {
+ Query->detach();
+ return Query->canStillFail();
+ });
+ auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols));
+ if (DeliverError)
+ Query->handleFailed(std::move(Err));
+ else
+ ES.reportError(std::move(Err));
+ }
+
+#if LLVM_ENABLE_THREADS
+ auto ResultFuture = PromisedResult.get_future();
+ auto Result = ResultFuture.get();
+
+ {
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ if (ResolutionError) {
+ // ReadyError will never be assigned. Consume the success value.
+ cantFail(std::move(ReadyError));
+ return std::move(ResolutionError);
+ }
+ }
+
+ if (WaitUntilReady) {
+ auto ReadyFuture = PromisedReady.get_future();
+ ReadyFuture.get();
+
+ {
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ if (ReadyError)
+ return std::move(ReadyError);
+ }
+ } else
+ cantFail(std::move(ReadyError));
+
+ return std::move(Result);
+
+#else
+ if (ResolutionError) {
+ // ReadyError will never be assigned. Consume the success value.
+ cantFail(std::move(ReadyError));
+ return std::move(ResolutionError);
+ }
+
+ if (ReadyError)
+ return std::move(ReadyError);
+
+ return Result;
+#endif
+}
+
+void ExecutionSessionBase::lookup(
+ const VSOList &VSOs, const SymbolNameSet &Symbols,
+ SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady,
+ RegisterDependenciesFunction RegisterDependencies) {
+
+ // lookup can be re-entered recursively if running on a single thread. Run any
+ // outstanding MUs in case this query depends on them, otherwise the main
+ // thread will starve waiting for a result from an MU that it failed to run.
+ runOutstandingMUs();
+
+ auto Unresolved = std::move(Symbols);
+ std::map<VSO *, MaterializationUnitList> MUsMap;
+ auto Q = std::make_shared<AsynchronousSymbolQuery>(
+ Symbols, std::move(OnResolve), std::move(OnReady));
+ bool QueryIsFullyResolved = false;
+ bool QueryIsFullyReady = false;
+ bool QueryFailed = false;
+
+ runSessionLocked([&]() {
+ for (auto *V : VSOs) {
+ assert(V && "VSOList entries must not be null");
+ assert(!MUsMap.count(V) &&
+ "VSOList should not contain duplicate entries");
+ V->lodgeQuery(Q, Unresolved, MUsMap[V]);
+ }
+
+ if (Unresolved.empty()) {
+ // Query lodged successfully.
+
+ // Record whether this query is fully ready / resolved. We will use
+ // this to call handleFullyResolved/handleFullyReady outside the session
+ // lock.
+ QueryIsFullyResolved = Q->isFullyResolved();
+ QueryIsFullyReady = Q->isFullyReady();
+
+ // Call the register dependencies function.
+ if (RegisterDependencies && !Q->QueryRegistrations.empty())
+ RegisterDependencies(Q->QueryRegistrations);
+ } else {
+ // Query failed due to unresolved symbols.
+ QueryFailed = true;
+
+ // Disconnect the query from its dependencies.
+ Q->detach();
+
+ // Replace the MUs.
+ for (auto &KV : MUsMap)
+ for (auto &MU : KV.second)
+ KV.first->replace(std::move(MU));
+ }
+ });
+
+ if (QueryFailed) {
+ Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved)));
+ return;
+ } else {
+ if (QueryIsFullyResolved)
+ Q->handleFullyResolved();
+ if (QueryIsFullyReady)
+ Q->handleFullyReady();
+ }
+
+ // Move the MUs to the OutstandingMUs list, then materialize.
+ {
+ std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
+
+ for (auto &KV : MUsMap)
+ for (auto &MU : KV.second)
+ OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU)));
+ }
+
+ runOutstandingMUs();
+}
+
+Expected<SymbolMap>
+ExecutionSessionBase::lookup(const VSOList &VSOs, const SymbolNameSet &Symbols,
+ RegisterDependenciesFunction RegisterDependencies,
+ bool WaitUntilReady) {
+#if LLVM_ENABLE_THREADS
+ // In the threaded case we use promises to return the results.
+ std::promise<SymbolMap> PromisedResult;
+ std::mutex ErrMutex;
+ Error ResolutionError = Error::success();
+ std::promise<void> PromisedReady;
+ Error ReadyError = Error::success();
+ auto OnResolve = [&](Expected<SymbolMap> R) {
+ if (R)
+ PromisedResult.set_value(std::move(*R));
+ else {
+ {
+ ErrorAsOutParameter _(&ResolutionError);
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ ResolutionError = R.takeError();
+ }
+ PromisedResult.set_value(SymbolMap());
+ }
+ };
+
+ std::function<void(Error)> OnReady;
+ if (WaitUntilReady) {
+ OnReady = [&](Error Err) {
+ if (Err) {
+ ErrorAsOutParameter _(&ReadyError);
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ ReadyError = std::move(Err);
+ }
+ PromisedReady.set_value();
+ };
+ } else {
+ OnReady = [&](Error Err) {
+ if (Err)
+ reportError(std::move(Err));
+ };
+ }
+
+#else
+ SymbolMap Result;
+ Error ResolutionError = Error::success();
+ Error ReadyError = Error::success();
+
+ auto OnResolve = [&](Expected<SymbolMap> R) {
+ ErrorAsOutParameter _(&ResolutionError);
+ if (R)
+ Result = std::move(*R);
+ else
+ ResolutionError = R.takeError();
+ };
+
+ std::function<void(Error)> OnReady;
+ if (WaitUntilReady) {
+ OnReady = [&](Error Err) {
+ ErrorAsOutParameter _(&ReadyError);
+ if (Err)
+ ReadyError = std::move(Err);
+ };
+ } else {
+ OnReady = [&](Error Err) {
+ if (Err)
+ reportError(std::move(Err));
+ };
+ }
+#endif
+
+ // Perform the asynchronous lookup.
+ lookup(VSOs, Symbols, OnResolve, OnReady, RegisterDependencies);
+
+#if LLVM_ENABLE_THREADS
+ auto ResultFuture = PromisedResult.get_future();
+ auto Result = ResultFuture.get();
+
+ {
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ if (ResolutionError) {
+ // ReadyError will never be assigned. Consume the success value.
+ cantFail(std::move(ReadyError));
+ return std::move(ResolutionError);
+ }
+ }
+
+ if (WaitUntilReady) {
+ auto ReadyFuture = PromisedReady.get_future();
+ ReadyFuture.get();
+
+ {
+ std::lock_guard<std::mutex> Lock(ErrMutex);
+ if (ReadyError)
+ return std::move(ReadyError);
+ }
+ } else
+ cantFail(std::move(ReadyError));
+
+ return std::move(Result);
+
+#else
+ if (ResolutionError) {
+ // ReadyError will never be assigned. Consume the success value.
+ cantFail(std::move(ReadyError));
+ return std::move(ResolutionError);
+ }
+
+ if (ReadyError)
+ return std::move(ReadyError);
+
+ return Result;
+#endif
+}
+
+void ExecutionSessionBase::runOutstandingMUs() {
+ while (1) {
+ std::pair<VSO *, std::unique_ptr<MaterializationUnit>> VSOAndMU;
+
+ {
+ std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
+ if (!OutstandingMUs.empty()) {
+ VSOAndMU = std::move(OutstandingMUs.back());
+ OutstandingMUs.pop_back();
+ }
+ }
+
+ if (VSOAndMU.first) {
+ assert(VSOAndMU.second && "VSO, but no MU?");
+ dispatchMaterialization(*VSOAndMU.first, std::move(VSOAndMU.second));
+ } else
+ break;
+ }
+}
+
+AsynchronousSymbolQuery::AsynchronousSymbolQuery(
+ const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved,
+ SymbolsReadyCallback NotifySymbolsReady)
+ : NotifySymbolsResolved(std::move(NotifySymbolsResolved)),
+ NotifySymbolsReady(std::move(NotifySymbolsReady)) {
+ NotYetResolvedCount = NotYetReadyCount = Symbols.size();
+
+ for (auto &S : Symbols)
+ ResolvedSymbols[S] = nullptr;
+}
+
+void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name,
+ JITEvaluatedSymbol Sym) {
+ auto I = ResolvedSymbols.find(Name);
+ assert(I != ResolvedSymbols.end() &&
+ "Resolving symbol outside the requested set");
+ assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name");
+ I->second = std::move(Sym);
+ --NotYetResolvedCount;
+}
+
+void AsynchronousSymbolQuery::handleFullyResolved() {
+ assert(NotYetResolvedCount == 0 && "Not fully resolved?");
+ assert(NotifySymbolsResolved &&
+ "NotifySymbolsResolved already called or error occurred");
+ NotifySymbolsResolved(std::move(ResolvedSymbols));
+ NotifySymbolsResolved = SymbolsResolvedCallback();
+}
+
+void AsynchronousSymbolQuery::notifySymbolReady() {
+ assert(NotYetReadyCount != 0 && "All symbols already finalized");
+ --NotYetReadyCount;
+}
+
+void AsynchronousSymbolQuery::handleFullyReady() {
+ assert(QueryRegistrations.empty() &&
+ "Query is still registered with some symbols");
+ assert(!NotifySymbolsResolved && "Resolution not applied yet");
+ NotifySymbolsReady(Error::success());
+ NotifySymbolsReady = SymbolsReadyCallback();
+}
+
+bool AsynchronousSymbolQuery::canStillFail() {
+ return (NotifySymbolsResolved || NotifySymbolsReady);
+}
+
+void AsynchronousSymbolQuery::handleFailed(Error Err) {
+ assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
+ NotYetResolvedCount == 0 && NotYetReadyCount == 0 &&
+ "Query should already have been abandoned");
+ if (NotifySymbolsResolved) {
+ NotifySymbolsResolved(std::move(Err));
+ NotifySymbolsResolved = SymbolsResolvedCallback();
+ } else {
+ assert(NotifySymbolsReady && "Failed after both callbacks issued?");
+ NotifySymbolsReady(std::move(Err));
+ }
+ NotifySymbolsReady = SymbolsReadyCallback();
+}
+
+void AsynchronousSymbolQuery::addQueryDependence(VSO &V, SymbolStringPtr Name) {
+ bool Added = QueryRegistrations[&V].insert(std::move(Name)).second;
+ (void)Added;
+ assert(Added && "Duplicate dependence notification?");
+}
+
+void AsynchronousSymbolQuery::removeQueryDependence(
+ VSO &V, const SymbolStringPtr &Name) {
+ auto QRI = QueryRegistrations.find(&V);
+ assert(QRI != QueryRegistrations.end() && "No dependencies registered for V");
+ assert(QRI->second.count(Name) && "No dependency on Name in V");
+ QRI->second.erase(Name);
+ if (QRI->second.empty())
+ QueryRegistrations.erase(QRI);
+}
+
+void AsynchronousSymbolQuery::detach() {
+ ResolvedSymbols.clear();
+ NotYetResolvedCount = 0;
+ NotYetReadyCount = 0;
+ for (auto &KV : QueryRegistrations)
+ KV.first->detachQueryHelper(*this, KV.second);
+ QueryRegistrations.clear();
+}
+
+MaterializationResponsibility::MaterializationResponsibility(
+ VSO &V, SymbolFlagsMap SymbolFlags)
+ : V(V), SymbolFlags(std::move(SymbolFlags)) {
+ assert(!this->SymbolFlags.empty() && "Materializing nothing?");
+
+#ifndef NDEBUG
+ for (auto &KV : this->SymbolFlags)
+ KV.second |= JITSymbolFlags::Materializing;
+#endif
+}
+
+MaterializationResponsibility::~MaterializationResponsibility() {
+ assert(SymbolFlags.empty() &&
+ "All symbols should have been explicitly materialized or failed");
+}
+
+SymbolNameSet MaterializationResponsibility::getRequestedSymbols() {
+ return V.getRequestedSymbols(SymbolFlags);
+}
+
+void MaterializationResponsibility::resolve(const SymbolMap &Symbols) {
+#ifndef NDEBUG
+ for (auto &KV : Symbols) {
+ auto I = SymbolFlags.find(KV.first);
+ assert(I != SymbolFlags.end() &&
+ "Resolving symbol outside this responsibility set");
+ assert(I->second.isMaterializing() && "Duplicate resolution");
+ I->second &= ~JITSymbolFlags::Materializing;
+ if (I->second.isWeak())
+ assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) &&
+ "Resolving symbol with incorrect flags");
+ else
+ assert(I->second == KV.second.getFlags() &&
+ "Resolving symbol with incorrect flags");
+ }
+#endif
+
+ V.resolve(Symbols);
+}
+
+void MaterializationResponsibility::finalize() {
+#ifndef NDEBUG
+ for (auto &KV : SymbolFlags)
+ assert(!KV.second.isMaterializing() &&
+ "Failed to resolve symbol before finalization");
+#endif // NDEBUG
+
+ V.finalize(SymbolFlags);
+ SymbolFlags.clear();
+}
+
+Error MaterializationResponsibility::defineMaterializing(
+ const SymbolFlagsMap &NewSymbolFlags) {
+ // Add the given symbols to this responsibility object.
+ // It's ok if we hit a duplicate here: In that case the new version will be
+ // discarded, and the VSO::defineMaterializing method will return a duplicate
+ // symbol error.
+ for (auto &KV : NewSymbolFlags) {
+ auto I = SymbolFlags.insert(KV).first;
+ (void)I;
+#ifndef NDEBUG
+ I->second |= JITSymbolFlags::Materializing;
+#endif
+ }
+
+ return V.defineMaterializing(NewSymbolFlags);
+}
+
+void MaterializationResponsibility::failMaterialization() {
+
+ SymbolNameSet FailedSymbols;
+ for (auto &KV : SymbolFlags)
+ FailedSymbols.insert(KV.first);
+
+ V.notifyFailed(FailedSymbols);
+ SymbolFlags.clear();
+}
+
+void MaterializationResponsibility::replace(
+ std::unique_ptr<MaterializationUnit> MU) {
+ for (auto &KV : MU->getSymbols())
+ SymbolFlags.erase(KV.first);
+
+ V.replace(std::move(MU));
+}
+
+MaterializationResponsibility
+MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) {
+ SymbolFlagsMap DelegatedFlags;
+
+ for (auto &Name : Symbols) {
+ auto I = SymbolFlags.find(Name);
+ assert(I != SymbolFlags.end() &&
+ "Symbol is not tracked by this MaterializationResponsibility "
+ "instance");
+
+ DelegatedFlags[Name] = std::move(I->second);
+ SymbolFlags.erase(I);
+ }
+
+ return MaterializationResponsibility(V, std::move(DelegatedFlags));
+}
+
+void MaterializationResponsibility::addDependencies(
+ const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) {
+ assert(SymbolFlags.count(Name) &&
+ "Symbol not covered by this MaterializationResponsibility instance");
+ V.addDependencies(Name, Dependencies);
+}
+
+void MaterializationResponsibility::addDependenciesForAll(
+ const SymbolDependenceMap &Dependencies) {
+ for (auto &KV : SymbolFlags)
+ V.addDependencies(KV.first, Dependencies);
+}
+
+AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit(
+ SymbolMap Symbols)
+ : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
+
+void AbsoluteSymbolsMaterializationUnit::materialize(
+ MaterializationResponsibility R) {
+ R.resolve(Symbols);
+ R.finalize();
+}
+
+void AbsoluteSymbolsMaterializationUnit::discard(const VSO &V,
+ SymbolStringPtr Name) {
+ assert(Symbols.count(Name) && "Symbol is not part of this MU");
+ Symbols.erase(Name);
+}
+
+SymbolFlagsMap
+AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
+ SymbolFlagsMap Flags;
+ for (const auto &KV : Symbols)
+ Flags[KV.first] = KV.second.getFlags();
+ return Flags;
+}
+
+ReExportsMaterializationUnit::ReExportsMaterializationUnit(
+ VSO *SourceVSO, SymbolAliasMap Aliases)
+ : MaterializationUnit(extractFlags(Aliases)), SourceVSO(SourceVSO),
+ Aliases(std::move(Aliases)) {}
+
+void ReExportsMaterializationUnit::materialize(
+ MaterializationResponsibility R) {
+
+ auto &ES = R.getTargetVSO().getExecutionSession();
+ VSO &TgtV = R.getTargetVSO();
+ VSO &SrcV = SourceVSO ? *SourceVSO : TgtV;
+
+ // Find the set of requested aliases and aliasees. Return any unrequested
+ // aliases back to the VSO so as to not prematurely materialize any aliasees.
+ auto RequestedSymbols = R.getRequestedSymbols();
+ SymbolAliasMap RequestedAliases;
+
+ for (auto &Name : RequestedSymbols) {
+ auto I = Aliases.find(Name);
+ assert(I != Aliases.end() && "Symbol not found in aliases map?");
+ RequestedAliases[Name] = std::move(I->second);
+ Aliases.erase(I);
+ }
+
+ if (!Aliases.empty()) {
+ if (SourceVSO)
+ R.replace(reexports(*SourceVSO, std::move(Aliases)));
+ else
+ R.replace(symbolAliases(std::move(Aliases)));
+ }
+
+ // The OnResolveInfo struct will hold the aliases and responsibilty for each
+ // query in the list.
+ struct OnResolveInfo {
+ OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases)
+ : R(std::move(R)), Aliases(std::move(Aliases)) {}
+
+ MaterializationResponsibility R;
+ SymbolAliasMap Aliases;
+ };
+
+ // Build a list of queries to issue. In each round we build the largest set of
+ // aliases that we can resolve without encountering a chain definition of the
+ // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would
+ // be waitin on a symbol that it itself had to resolve. Usually this will just
+ // involve one round and a single query.
+
+ std::vector<std::pair<SymbolNameSet, std::shared_ptr<OnResolveInfo>>>
+ QueryInfos;
+ while (!RequestedAliases.empty()) {
+ SymbolNameSet ResponsibilitySymbols;
+ SymbolNameSet QuerySymbols;
+ SymbolAliasMap QueryAliases;
+
+ for (auto I = RequestedAliases.begin(), E = RequestedAliases.end();
+ I != E;) {
+ auto Tmp = I++;
+
+ // Chain detected. Skip this symbol for this round.
+ if (&SrcV == &TgtV && (QueryAliases.count(Tmp->second.Aliasee) ||
+ RequestedAliases.count(Tmp->second.Aliasee)))
+ continue;
+
+ ResponsibilitySymbols.insert(Tmp->first);
+ QuerySymbols.insert(Tmp->second.Aliasee);
+ QueryAliases[Tmp->first] = std::move(Tmp->second);
+ RequestedAliases.erase(Tmp);
+ }
+ assert(!QuerySymbols.empty() && "Alias cycle detected!");
+
+ auto QueryInfo = std::make_shared<OnResolveInfo>(
+ R.delegate(ResponsibilitySymbols), std::move(QueryAliases));
+ QueryInfos.push_back(
+ make_pair(std::move(QuerySymbols), std::move(QueryInfo)));
+ }
+
+ // Issue the queries.
+ while (!QueryInfos.empty()) {
+ auto QuerySymbols = std::move(QueryInfos.back().first);
+ auto QueryInfo = std::move(QueryInfos.back().second);
+
+ QueryInfos.pop_back();
+
+ auto RegisterDependencies = [QueryInfo,
+ &SrcV](const SymbolDependenceMap &Deps) {
+ // If there were no materializing symbols, just bail out.
+ if (Deps.empty())
+ return;
+
+ // Otherwise the only deps should be on SrcV.
+ assert(Deps.size() == 1 && Deps.count(&SrcV) &&
+ "Unexpected dependencies for reexports");
+
+ auto &SrcVDeps = Deps.find(&SrcV)->second;
+ SymbolDependenceMap PerAliasDepsMap;
+ auto &PerAliasDeps = PerAliasDepsMap[&SrcV];
+
+ for (auto &KV : QueryInfo->Aliases)
+ if (SrcVDeps.count(KV.second.Aliasee)) {
+ PerAliasDeps = {KV.second.Aliasee};
+ QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap);
+ }
+ };
+
+ auto OnResolve = [QueryInfo](Expected<SymbolMap> Result) {
+ if (Result) {
+ SymbolMap ResolutionMap;
+ for (auto &KV : QueryInfo->Aliases) {
+ assert(Result->count(KV.second.Aliasee) &&
+ "Result map missing entry?");
+ ResolutionMap[KV.first] = JITEvaluatedSymbol(
+ (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags);
+ }
+ QueryInfo->R.resolve(ResolutionMap);
+ QueryInfo->R.finalize();
+ } else {
+ auto &ES = QueryInfo->R.getTargetVSO().getExecutionSession();
+ ES.reportError(Result.takeError());
+ QueryInfo->R.failMaterialization();
+ }
+ };
+
+ auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); };
+
+ ES.lookup({&SrcV}, QuerySymbols, std::move(OnResolve), std::move(OnReady),
+ std::move(RegisterDependencies));
+ }
+}
+
+void ReExportsMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) {
+ assert(Aliases.count(Name) &&
+ "Symbol not covered by this MaterializationUnit");
+ Aliases.erase(Name);
+}
+
+SymbolFlagsMap
+ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
+ SymbolFlagsMap SymbolFlags;
+ for (auto &KV : Aliases)
+ SymbolFlags[KV.first] = KV.second.AliasFlags;
+
+ return SymbolFlags;
+}
+
+Expected<SymbolAliasMap>
+buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols) {
+ auto Flags = SourceV.lookupFlags(Symbols);
+
+ if (Flags.size() != Symbols.size()) {
+ SymbolNameSet Unresolved = Symbols;
+ for (auto &KV : Flags)
+ Unresolved.erase(KV.first);
+ return make_error<SymbolsNotFound>(std::move(Unresolved));
+ }
+
+ SymbolAliasMap Result;
+ for (auto &Name : Symbols) {
+ assert(Flags.count(Name) && "Missing entry in flags map");
+ Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]);
+ }
+
+ return Result;
+}
+
+Error VSO::defineMaterializing(const SymbolFlagsMap &SymbolFlags) {
+ return ES.runSessionLocked([&]() -> Error {
+ std::vector<SymbolMap::iterator> AddedSyms;
+
+ for (auto &KV : SymbolFlags) {
+ SymbolMap::iterator EntryItr;
+ bool Added;
+
+ auto NewFlags = KV.second;
+ NewFlags |= JITSymbolFlags::Materializing;
+
+ std::tie(EntryItr, Added) = Symbols.insert(
+ std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags)));
+
+ if (Added)
+ AddedSyms.push_back(EntryItr);
+ else {
+ // Remove any symbols already added.
+ for (auto &SI : AddedSyms)
+ Symbols.erase(SI);
+
+ // FIXME: Return all duplicates.
+ return make_error<DuplicateDefinition>(*KV.first);
+ }
+ }
+
+ return Error::success();
+ });
+}
+
+void VSO::replace(std::unique_ptr<MaterializationUnit> MU) {
+ assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
+
+ auto MustRunMU =
+ ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> {
+
+#ifndef NDEBUG
+ for (auto &KV : MU->getSymbols()) {
+ auto SymI = Symbols.find(KV.first);
+ assert(SymI != Symbols.end() && "Replacing unknown symbol");
+ assert(!SymI->second.getFlags().isLazy() &&
+ SymI->second.getFlags().isMaterializing() &&
+ "Can not replace symbol that is not materializing");
+ assert(UnmaterializedInfos.count(KV.first) == 0 &&
+ "Symbol being replaced should have no UnmaterializedInfo");
+ }
+#endif // NDEBUG
+
+ // If any symbol has pending queries against it then we need to
+ // materialize MU immediately.
+ for (auto &KV : MU->getSymbols()) {
+ auto MII = MaterializingInfos.find(KV.first);
+ if (MII != MaterializingInfos.end()) {
+ if (!MII->second.PendingQueries.empty())
+ return std::move(MU);
+ }
+ }
+
+ // Otherwise, make MU responsible for all the symbols.
+ auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU));
+ for (auto &KV : UMI->MU->getSymbols()) {
+ assert(!KV.second.isLazy() &&
+ "Lazy flag should be managed internally.");
+ assert(!KV.second.isMaterializing() &&
+ "Materializing flags should be managed internally.");
+
+ auto SymI = Symbols.find(KV.first);
+ JITSymbolFlags ReplaceFlags = KV.second;
+ ReplaceFlags |= JITSymbolFlags::Lazy;
+ SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(),
+ std::move(ReplaceFlags));
+ UnmaterializedInfos[KV.first] = UMI;
+ }
+
+ return nullptr;
+ });
+
+ if (MustRunMU)
+ ES.dispatchMaterialization(*this, std::move(MustRunMU));
+}
+
+SymbolNameSet VSO::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) {
+ return ES.runSessionLocked([&]() {
+ SymbolNameSet RequestedSymbols;
+
+ for (auto &KV : SymbolFlags) {
+ assert(Symbols.count(KV.first) && "VSO does not cover this symbol?");
+ assert(Symbols[KV.first].getFlags().isMaterializing() &&
+ "getRequestedSymbols can only be called for materializing "
+ "symbols");
+ auto I = MaterializingInfos.find(KV.first);
+ if (I == MaterializingInfos.end())
+ continue;
+
+ if (!I->second.PendingQueries.empty())
+ RequestedSymbols.insert(KV.first);
+ }
+
+ return RequestedSymbols;
+ });
+}
+
+void VSO::addDependencies(const SymbolStringPtr &Name,
+ const SymbolDependenceMap &Dependencies) {
+ assert(Symbols.count(Name) && "Name not in symbol table");
+ assert((Symbols[Name].getFlags().isLazy() ||
+ Symbols[Name].getFlags().isMaterializing()) &&
+ "Symbol is not lazy or materializing");
+
+ auto &MI = MaterializingInfos[Name];
+ assert(!MI.IsFinalized && "Can not add dependencies to finalized symbol");
+
+ for (auto &KV : Dependencies) {
+ assert(KV.first && "Null VSO in dependency?");
+ auto &OtherVSO = *KV.first;
+ auto &DepsOnOtherVSO = MI.UnfinalizedDependencies[&OtherVSO];
+
+ for (auto &OtherSymbol : KV.second) {
+#ifndef NDEBUG
+ // Assert that this symbol exists and has not been finalized already.
+ auto SymI = OtherVSO.Symbols.find(OtherSymbol);
+ assert(SymI != OtherVSO.Symbols.end() &&
+ (SymI->second.getFlags().isLazy() ||
+ SymI->second.getFlags().isMaterializing()) &&
+ "Dependency on finalized symbol");
+#endif
+
+ auto &OtherMI = OtherVSO.MaterializingInfos[OtherSymbol];
+
+ if (OtherMI.IsFinalized)
+ transferFinalizedNodeDependencies(MI, Name, OtherMI);
+ else if (&OtherVSO != this || OtherSymbol != Name) {
+ OtherMI.Dependants[this].insert(Name);
+ DepsOnOtherVSO.insert(OtherSymbol);
+ }
+ }
+
+ if (DepsOnOtherVSO.empty())
+ MI.UnfinalizedDependencies.erase(&OtherVSO);
+ }
+}
+
+void VSO::resolve(const SymbolMap &Resolved) {
+ auto FullyResolvedQueries = ES.runSessionLocked([&, this]() {
+ AsynchronousSymbolQuerySet FullyResolvedQueries;
+ for (const auto &KV : Resolved) {
+ auto &Name = KV.first;
+ auto Sym = KV.second;
+
+ assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() &&
+ "Materializing flags should be managed internally");
+
+ auto I = Symbols.find(Name);
+
+ assert(I != Symbols.end() && "Symbol not found");
+ assert(!I->second.getFlags().isLazy() &&
+ I->second.getFlags().isMaterializing() &&
+ "Symbol should be materializing");
+ assert(I->second.getAddress() == 0 && "Symbol has already been resolved");
+
+ assert((Sym.getFlags() & ~JITSymbolFlags::Weak) ==
+ (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) &
+ ~JITSymbolFlags::Weak) &&
+ "Resolved flags should match the declared flags");
+
+ // Once resolved, symbols can never be weak.
+ JITSymbolFlags ResolvedFlags = Sym.getFlags();
+ ResolvedFlags &= ~JITSymbolFlags::Weak;
+ ResolvedFlags |= JITSymbolFlags::Materializing;
+ I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags);
+
+ auto &MI = MaterializingInfos[Name];
+ for (auto &Q : MI.PendingQueries) {
+ Q->resolve(Name, Sym);
+ if (Q->isFullyResolved())
+ FullyResolvedQueries.insert(Q);
+ }
+ }
+
+ return FullyResolvedQueries;
+ });
+
+ for (auto &Q : FullyResolvedQueries) {
+ assert(Q->isFullyResolved() && "Q not fully resolved");
+ Q->handleFullyResolved();
+ }
+}
+
+void VSO::finalize(const SymbolFlagsMap &Finalized) {
+ auto FullyReadyQueries = ES.runSessionLocked([&, this]() {
+ AsynchronousSymbolQuerySet ReadyQueries;
+
+ for (const auto &KV : Finalized) {
+ const auto &Name = KV.first;
+
+ auto MII = MaterializingInfos.find(Name);
+ assert(MII != MaterializingInfos.end() &&
+ "Missing MaterializingInfo entry");
+
+ auto &MI = MII->second;
+
+ // For each dependant, transfer this node's unfinalized dependencies to
+ // it. If the dependant node is fully finalized then notify any pending
+ // queries.
+ for (auto &KV : MI.Dependants) {
+ auto &DependantVSO = *KV.first;
+ for (auto &DependantName : KV.second) {
+ auto DependantMII =
+ DependantVSO.MaterializingInfos.find(DependantName);
+ assert(DependantMII != DependantVSO.MaterializingInfos.end() &&
+ "Dependant should have MaterializingInfo");
+
+ auto &DependantMI = DependantMII->second;
+
+ // Remove the dependant's dependency on this node.
+ assert(DependantMI.UnfinalizedDependencies[this].count(Name) &&
+ "Dependant does not count this symbol as a dependency?");
+ DependantMI.UnfinalizedDependencies[this].erase(Name);
+ if (DependantMI.UnfinalizedDependencies[this].empty())
+ DependantMI.UnfinalizedDependencies.erase(this);
+
+ // Transfer unfinalized dependencies from this node to the dependant.
+ DependantVSO.transferFinalizedNodeDependencies(DependantMI,
+ DependantName, MI);
+
+ // If the dependant is finalized and this node was the last of its
+ // unfinalized dependencies then notify any pending queries on the
+ // dependant node.
+ if (DependantMI.IsFinalized &&
+ DependantMI.UnfinalizedDependencies.empty()) {
+ assert(DependantMI.Dependants.empty() &&
+ "Dependants should be empty by now");
+ for (auto &Q : DependantMI.PendingQueries) {
+ Q->notifySymbolReady();
+ if (Q->isFullyReady())
+ ReadyQueries.insert(Q);
+ Q->removeQueryDependence(DependantVSO, DependantName);
+ }
+
+ // If this dependant node was fully finalized we can erase its
+ // MaterializingInfo and update its materializing state.
+ assert(DependantVSO.Symbols.count(DependantName) &&
+ "Dependant has no entry in the Symbols table");
+ auto &DependantSym = DependantVSO.Symbols[DependantName];
+ DependantSym.setFlags(static_cast<JITSymbolFlags::FlagNames>(
+ DependantSym.getFlags() & ~JITSymbolFlags::Materializing));
+ DependantVSO.MaterializingInfos.erase(DependantMII);
+ }
+ }
+ }
+ MI.Dependants.clear();
+ MI.IsFinalized = true;
+
+ if (MI.UnfinalizedDependencies.empty()) {
+ for (auto &Q : MI.PendingQueries) {
+ Q->notifySymbolReady();
+ if (Q->isFullyReady())
+ ReadyQueries.insert(Q);
+ Q->removeQueryDependence(*this, Name);
+ }
+ assert(Symbols.count(Name) &&
+ "Symbol has no entry in the Symbols table");
+ auto &Sym = Symbols[Name];
+ Sym.setFlags(static_cast<JITSymbolFlags::FlagNames>(
+ Sym.getFlags() & ~JITSymbolFlags::Materializing));
+ MaterializingInfos.erase(MII);
+ }
+ }
+
+ return ReadyQueries;
+ });
+
+ for (auto &Q : FullyReadyQueries) {
+ assert(Q->isFullyReady() && "Q is not fully ready");
+ Q->handleFullyReady();
+ }
+}
+
+void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) {
+
+ // FIXME: This should fail any transitively dependant symbols too.
+
+ auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() {
+ AsynchronousSymbolQuerySet FailedQueries;
+
+ for (auto &Name : FailedSymbols) {
+ auto I = Symbols.find(Name);
+ assert(I != Symbols.end() && "Symbol not present in this VSO");
+ Symbols.erase(I);
+
+ auto MII = MaterializingInfos.find(Name);
+
+ // If we have not created a MaterializingInfo for this symbol yet then
+ // there is nobody to notify.
+ if (MII == MaterializingInfos.end())
+ continue;
+
+ // Copy all the queries to the FailedQueries list, then abandon them.
+ // This has to be a copy, and the copy has to come before the abandon
+ // operation: Each Q.detach() call will reach back into this
+ // PendingQueries list to remove Q.
+ for (auto &Q : MII->second.PendingQueries)
+ FailedQueries.insert(Q);
+
+ for (auto &Q : FailedQueries)
+ Q->detach();
+
+ assert(MII->second.PendingQueries.empty() &&
+ "Queries remain after symbol was failed");
+
+ MaterializingInfos.erase(MII);
+ }
+
+ return FailedQueries;
+ });
+
+ for (auto &Q : FailedQueriesToNotify)
+ Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols));
+}
+
+void VSO::setSearchOrder(VSOList NewSearchOrder, bool SearchThisVSOFirst) {
+ if (SearchThisVSOFirst && NewSearchOrder.front() != this)
+ NewSearchOrder.insert(NewSearchOrder.begin(), this);
+
+ ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); });
+}
+
+void VSO::addToSearchOrder(VSO &V) {
+ ES.runSessionLocked([&]() { SearchOrder.push_back(&V); });
+}
+
+void VSO::replaceInSearchOrder(VSO &OldV, VSO &NewV) {
+ ES.runSessionLocked([&]() {
+ auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &OldV);
+
+ if (I != SearchOrder.end())
+ *I = &NewV;
+ });
+}
+
+void VSO::removeFromSearchOrder(VSO &V) {
+ ES.runSessionLocked([&]() {
+ auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &V);
+ if (I != SearchOrder.end())
+ SearchOrder.erase(I);
+ });
+}
+
+SymbolFlagsMap VSO::lookupFlags(const SymbolNameSet &Names) {
+ return ES.runSessionLocked([&, this]() {
+ SymbolFlagsMap Result;
+ auto Unresolved = lookupFlagsImpl(Result, Names);
+ if (FallbackDefinitionGenerator && !Unresolved.empty()) {
+ auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved);
+ if (!FallbackDefs.empty()) {
+ auto Unresolved2 = lookupFlagsImpl(Result, FallbackDefs);
+ (void)Unresolved2;
+ assert(Unresolved2.empty() &&
+ "All fallback defs should have been found by lookupFlagsImpl");
+ }
+ };
+ return Result;
+ });
+}
+
+SymbolNameSet VSO::lookupFlagsImpl(SymbolFlagsMap &Flags,
+ const SymbolNameSet &Names) {
+ SymbolNameSet Unresolved;
+
+ for (auto &Name : Names) {
+ auto I = Symbols.find(Name);
+
+ if (I == Symbols.end()) {
+ Unresolved.insert(Name);
+ continue;
+ }
+
+ assert(!Flags.count(Name) && "Symbol already present in Flags map");
+ Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags());
+ }
+
+ return Unresolved;
+}
+
+void VSO::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q,
+ SymbolNameSet &Unresolved, MaterializationUnitList &MUs) {
+ assert(Q && "Query can not be null");
+
+ lodgeQueryImpl(Q, Unresolved, MUs);
+ if (FallbackDefinitionGenerator && !Unresolved.empty()) {
+ auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved);
+ if (!FallbackDefs.empty()) {
+ for (auto &D : FallbackDefs)
+ Unresolved.erase(D);
+ lodgeQueryImpl(Q, FallbackDefs, MUs);
+ assert(FallbackDefs.empty() &&
+ "All fallback defs should have been found by lookupImpl");
+ }
+ }
+}
+
+void VSO::lodgeQueryImpl(
+ std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved,
+ std::vector<std::unique_ptr<MaterializationUnit>> &MUs) {
+ for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) {
+ auto TmpI = I++;
+ auto Name = *TmpI;
+
+ // Search for the name in Symbols. Skip it if not found.
+ auto SymI = Symbols.find(Name);
+ if (SymI == Symbols.end())
+ continue;
+
+ // If we found Name in V, remove it frome the Unresolved set and add it
+ // to the added set.
+ Unresolved.erase(TmpI);
+
+ // If the symbol has an address then resolve it.
+ if (SymI->second.getAddress() != 0)
+ Q->resolve(Name, SymI->second);
+
+ // If the symbol is lazy, get the MaterialiaztionUnit for it.
+ if (SymI->second.getFlags().isLazy()) {
+ assert(SymI->second.getAddress() == 0 &&
+ "Lazy symbol should not have a resolved address");
+ assert(!SymI->second.getFlags().isMaterializing() &&
+ "Materializing and lazy should not both be set");
+ auto UMII = UnmaterializedInfos.find(Name);
+ assert(UMII != UnmaterializedInfos.end() &&
+ "Lazy symbol should have UnmaterializedInfo");
+ auto MU = std::move(UMII->second->MU);
+ assert(MU != nullptr && "Materializer should not be null");
+
+ // Move all symbols associated with this MaterializationUnit into
+ // materializing state.
+ for (auto &KV : MU->getSymbols()) {
+ auto SymK = Symbols.find(KV.first);
+ auto Flags = SymK->second.getFlags();
+ Flags &= ~JITSymbolFlags::Lazy;
+ Flags |= JITSymbolFlags::Materializing;
+ SymK->second.setFlags(Flags);
+ UnmaterializedInfos.erase(KV.first);
+ }
+
+ // Add MU to the list of MaterializationUnits to be materialized.
+ MUs.push_back(std::move(MU));
+ } else if (!SymI->second.getFlags().isMaterializing()) {
+ // The symbol is neither lazy nor materializing. Finalize it and
+ // continue.
+ Q->notifySymbolReady();
+ continue;
+ }
+
+ // Add the query to the PendingQueries list.
+ assert(SymI->second.getFlags().isMaterializing() &&
+ "By this line the symbol should be materializing");
+ auto &MI = MaterializingInfos[Name];
+ MI.PendingQueries.push_back(Q);
+ Q->addQueryDependence(*this, Name);
+ }
+}
+
+SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q,
+ SymbolNameSet Names) {
+ assert(Q && "Query can not be null");
+
+ ES.runOutstandingMUs();
+
+ LookupImplActionFlags ActionFlags = None;
+ std::vector<std::unique_ptr<MaterializationUnit>> MUs;
+
+ SymbolNameSet Unresolved = std::move(Names);
+ ES.runSessionLocked([&, this]() {
+ ActionFlags = lookupImpl(Q, MUs, Unresolved);
+ if (FallbackDefinitionGenerator && !Unresolved.empty()) {
+ assert(ActionFlags == None &&
+ "ActionFlags set but unresolved symbols remain?");
+ auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved);
+ if (!FallbackDefs.empty()) {
+ for (auto &D : FallbackDefs)
+ Unresolved.erase(D);
+ ActionFlags = lookupImpl(Q, MUs, FallbackDefs);
+ assert(FallbackDefs.empty() &&
+ "All fallback defs should have been found by lookupImpl");
+ }
+ }
+ });
+
+ assert((MUs.empty() || ActionFlags == None) &&
+ "If action flags are set, there should be no work to do (so no MUs)");
+
+ if (ActionFlags & NotifyFullyResolved)
+ Q->handleFullyResolved();
+
+ if (ActionFlags & NotifyFullyReady)
+ Q->handleFullyReady();
+
+ // FIXME: Swap back to the old code below once RuntimeDyld works with
+ // callbacks from asynchronous queries.
+ // Add MUs to the OutstandingMUs list.
+ {
+ std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex);
+ for (auto &MU : MUs)
+ ES.OutstandingMUs.push_back(make_pair(this, std::move(MU)));
+ }
+ ES.runOutstandingMUs();
+
+ // Dispatch any required MaterializationUnits for materialization.
+ // for (auto &MU : MUs)
+ // ES.dispatchMaterialization(*this, std::move(MU));
+
+ return Unresolved;
+}
+
+VSO::LookupImplActionFlags
+VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q,
+ std::vector<std::unique_ptr<MaterializationUnit>> &MUs,
+ SymbolNameSet &Unresolved) {
+ LookupImplActionFlags ActionFlags = None;
+
+ for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) {
+ auto TmpI = I++;
+ auto Name = *TmpI;
+
+ // Search for the name in Symbols. Skip it if not found.
+ auto SymI = Symbols.find(Name);
+ if (SymI == Symbols.end())
+ continue;
+
+ // If we found Name in V, remove it frome the Unresolved set and add it
+ // to the dependencies set.
+ Unresolved.erase(TmpI);
+
+ // If the symbol has an address then resolve it.
+ if (SymI->second.getAddress() != 0) {
+ Q->resolve(Name, SymI->second);
+ if (Q->isFullyResolved())
+ ActionFlags |= NotifyFullyResolved;
+ }
+
+ // If the symbol is lazy, get the MaterialiaztionUnit for it.
+ if (SymI->second.getFlags().isLazy()) {
+ assert(SymI->second.getAddress() == 0 &&
+ "Lazy symbol should not have a resolved address");
+ assert(!SymI->second.getFlags().isMaterializing() &&
+ "Materializing and lazy should not both be set");
+ auto UMII = UnmaterializedInfos.find(Name);
+ assert(UMII != UnmaterializedInfos.end() &&
+ "Lazy symbol should have UnmaterializedInfo");
+ auto MU = std::move(UMII->second->MU);
+ assert(MU != nullptr && "Materializer should not be null");
+
+ // Kick all symbols associated with this MaterializationUnit into
+ // materializing state.
+ for (auto &KV : MU->getSymbols()) {
+ auto SymK = Symbols.find(KV.first);
+ auto Flags = SymK->second.getFlags();
+ Flags &= ~JITSymbolFlags::Lazy;
+ Flags |= JITSymbolFlags::Materializing;
+ SymK->second.setFlags(Flags);
+ UnmaterializedInfos.erase(KV.first);
+ }
+
+ // Add MU to the list of MaterializationUnits to be materialized.
+ MUs.push_back(std::move(MU));
+ } else if (!SymI->second.getFlags().isMaterializing()) {
+ // The symbol is neither lazy nor materializing. Finalize it and
+ // continue.
+ Q->notifySymbolReady();
+ if (Q->isFullyReady())
+ ActionFlags |= NotifyFullyReady;
+ continue;
+ }
+
+ // Add the query to the PendingQueries list.
+ assert(SymI->second.getFlags().isMaterializing() &&
+ "By this line the symbol should be materializing");
+ auto &MI = MaterializingInfos[Name];
+ MI.PendingQueries.push_back(Q);
+ Q->addQueryDependence(*this, Name);
+ }
+
+ return ActionFlags;
+}
+
+void VSO::dump(raw_ostream &OS) {
+ ES.runSessionLocked([&, this]() {
+ OS << "VSO \"" << VSOName
+ << "\" (ES: " << format("0x%016x", reinterpret_cast<uintptr_t>(&ES))
+ << "):\n"
+ << "Symbol table:\n";
+
+ for (auto &KV : Symbols) {
+ OS << " \"" << *KV.first
+ << "\": " << format("0x%016x", KV.second.getAddress());
+ if (KV.second.getFlags().isLazy() ||
+ KV.second.getFlags().isMaterializing()) {
+ OS << " (";
+ if (KV.second.getFlags().isLazy()) {
+ auto I = UnmaterializedInfos.find(KV.first);
+ assert(I != UnmaterializedInfos.end() &&
+ "Lazy symbol should have UnmaterializedInfo");
+ OS << " Lazy (MU=" << I->second->MU.get() << ")";
+ }
+ if (KV.second.getFlags().isMaterializing())
+ OS << " Materializing";
+ OS << " )\n";
+ } else
+ OS << "\n";
+ }
+
+ if (!MaterializingInfos.empty())
+ OS << " MaterializingInfos entries:\n";
+ for (auto &KV : MaterializingInfos) {
+ OS << " \"" << *KV.first << "\":\n"
+ << " IsFinalized = " << (KV.second.IsFinalized ? "true" : "false")
+ << "\n"
+ << " " << KV.second.PendingQueries.size()
+ << " pending queries: { ";
+ for (auto &Q : KV.second.PendingQueries)
+ OS << Q.get() << " ";
+ OS << "}\n Dependants:\n";
+ for (auto &KV2 : KV.second.Dependants)
+ OS << " " << KV2.first->getName() << ": " << KV2.second << "\n";
+ OS << " Unfinalized Dependencies:\n";
+ for (auto &KV2 : KV.second.UnfinalizedDependencies)
+ OS << " " << KV2.first->getName() << ": " << KV2.second << "\n";
+ }
+ });
+}
+
+VSO::VSO(ExecutionSessionBase &ES, std::string Name)
+ : ES(ES), VSOName(std::move(Name)) {
+ SearchOrder.push_back(this);
+}
+
+Error VSO::defineImpl(MaterializationUnit &MU) {
+ SymbolNameSet Duplicates;
+ SymbolNameSet MUDefsOverridden;
+
+ struct ExistingDefOverriddenEntry {
+ SymbolMap::iterator ExistingDefItr;
+ JITSymbolFlags NewFlags;
+ };
+ std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden;
+
+ for (auto &KV : MU.getSymbols()) {
+ assert(!KV.second.isLazy() && "Lazy flag should be managed internally.");
+ assert(!KV.second.isMaterializing() &&
+ "Materializing flags should be managed internally.");
+
+ SymbolMap::iterator EntryItr;
+ bool Added;
+
+ auto NewFlags = KV.second;
+ NewFlags |= JITSymbolFlags::Lazy;
+
+ std::tie(EntryItr, Added) = Symbols.insert(
+ std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags)));
+
+ if (!Added) {
+ if (KV.second.isStrong()) {
+ if (EntryItr->second.getFlags().isStrong() ||
+ (EntryItr->second.getFlags() & JITSymbolFlags::Materializing))
+ Duplicates.insert(KV.first);
+ else
+ ExistingDefsOverridden.push_back({EntryItr, NewFlags});
+ } else
+ MUDefsOverridden.insert(KV.first);
+ }
+ }
+
+ if (!Duplicates.empty()) {
+ // We need to remove the symbols we added.
+ for (auto &KV : MU.getSymbols()) {
+ if (Duplicates.count(KV.first))
+ continue;
+
+ bool Found = false;
+ for (const auto &EDO : ExistingDefsOverridden)
+ if (EDO.ExistingDefItr->first == KV.first)
+ Found = true;
+
+ if (!Found)
+ Symbols.erase(KV.first);
+ }
+
+ // FIXME: Return all duplicates.
+ return make_error<DuplicateDefinition>(**Duplicates.begin());
+ }
+
+ // Update flags on existing defs and call discard on their materializers.
+ for (auto &EDO : ExistingDefsOverridden) {
+ assert(EDO.ExistingDefItr->second.getFlags().isLazy() &&
+ !EDO.ExistingDefItr->second.getFlags().isMaterializing() &&
+ "Overridden existing def should be in the Lazy state");
+
+ EDO.ExistingDefItr->second.setFlags(EDO.NewFlags);
+
+ auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first);
+ assert(UMII != UnmaterializedInfos.end() &&
+ "Overridden existing def should have an UnmaterializedInfo");
+
+ UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first);
+ }
+
+ // Discard overridden symbols povided by MU.
+ for (auto &Sym : MUDefsOverridden)
+ MU.doDiscard(*this, Sym);
+
+ return Error::success();
+}
+
+void VSO::detachQueryHelper(AsynchronousSymbolQuery &Q,
+ const SymbolNameSet &QuerySymbols) {
+ for (auto &QuerySymbol : QuerySymbols) {
+ assert(MaterializingInfos.count(QuerySymbol) &&
+ "QuerySymbol does not have MaterializingInfo");
+ auto &MI = MaterializingInfos[QuerySymbol];
+
+ auto IdenticalQuery =
+ [&](const std::shared_ptr<AsynchronousSymbolQuery> &R) {
+ return R.get() == &Q;
+ };
+
+ auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(),
+ IdenticalQuery);
+ assert(I != MI.PendingQueries.end() &&
+ "Query Q should be in the PendingQueries list for QuerySymbol");
+ MI.PendingQueries.erase(I);
+ }
+}
+
+void VSO::transferFinalizedNodeDependencies(
+ MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName,
+ MaterializingInfo &FinalizedMI) {
+ for (auto &KV : FinalizedMI.UnfinalizedDependencies) {
+ auto &DependencyVSO = *KV.first;
+ SymbolNameSet *UnfinalizedDependenciesOnDependencyVSO = nullptr;
+
+ for (auto &DependencyName : KV.second) {
+ auto &DependencyMI = DependencyVSO.MaterializingInfos[DependencyName];
+
+ // Do not add self dependencies.
+ if (&DependencyMI == &DependantMI)
+ continue;
+
+ // If we haven't looked up the dependencies for DependencyVSO yet, do it
+ // now and cache the result.
+ if (!UnfinalizedDependenciesOnDependencyVSO)
+ UnfinalizedDependenciesOnDependencyVSO =
+ &DependantMI.UnfinalizedDependencies[&DependencyVSO];
+
+ DependencyMI.Dependants[this].insert(DependantName);
+ UnfinalizedDependenciesOnDependencyVSO->insert(DependencyName);
+ }
+ }
+}
+
+VSO &ExecutionSession::createVSO(std::string Name) {
+ return runSessionLocked([&, this]() -> VSO & {
+ VSOs.push_back(std::unique_ptr<VSO>(new VSO(*this, std::move(Name))));
+ return *VSOs.back();
+ });
+}
+
+Expected<SymbolMap> lookup(const VSOList &VSOs, SymbolNameSet Names) {
+
+ if (VSOs.empty())
+ return SymbolMap();
+
+ auto &ES = (*VSOs.begin())->getExecutionSession();
+
+ return ES.lookup(VSOs, Names, NoDependenciesToRegister, true);
+}
+
+/// Look up a symbol by searching a list of VSOs.
+Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name) {
+ SymbolNameSet Names({Name});
+ if (auto ResultMap = lookup(VSOs, std::move(Names))) {
+ assert(ResultMap->size() == 1 && "Unexpected number of results");
+ assert(ResultMap->count(Name) && "Missing result for symbol");
+ return std::move(ResultMap->begin()->second);
+ } else
+ return ResultMap.takeError();
+}
+
+MangleAndInterner::MangleAndInterner(ExecutionSessionBase &ES,
+ const DataLayout &DL)
+ : ES(ES), DL(DL) {}
+
+SymbolStringPtr MangleAndInterner::operator()(StringRef Name) {
+ std::string MangledName;
+ {
+ raw_string_ostream MangledNameStream(MangledName);
+ Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
+ }
+ return ES.getSymbolStringPool().intern(MangledName);
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index b7220dba88e9..6157677ce355 100644
--- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -13,10 +13,51 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Module.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
namespace llvm {
namespace orc {
+JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT)
+ : TT(std::move(TT)) {}
+
+Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() {
+ return JITTargetMachineBuilder(Triple(sys::getProcessTriple()));
+}
+
+Expected<std::unique_ptr<TargetMachine>>
+JITTargetMachineBuilder::createTargetMachine() {
+ if (!Arch.empty()) {
+ Triple::ArchType Type = Triple::getArchTypeForLLVMName(Arch);
+
+ if (Type == Triple::UnknownArch)
+ return make_error<StringError>(std::string("Unknown arch: ") + Arch,
+ inconvertibleErrorCode());
+ }
+
+ std::string ErrMsg;
+ auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg);
+ if (!TheTarget)
+ return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
+
+ auto *TM =
+ TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(),
+ Options, RM, CM, OptLevel, /*JIT*/ true);
+ if (!TM)
+ return make_error<StringError>("Could not allocate target machine",
+ inconvertibleErrorCode());
+
+ return std::unique_ptr<TargetMachine>(TM);
+}
+
+JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures(
+ const std::vector<std::string> &FeatureVec) {
+ for (const auto &F : FeatureVec)
+ Features.AddFeature(F);
+ return *this;
+}
+
CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End)
: InitList(
GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr),
@@ -67,7 +108,9 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const {
}
ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
- Value *Data = CS->getOperand(2);
+ Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr;
+ if (Data && !isa<GlobalValue>(Data))
+ Data = nullptr;
return Element(Priority->getZExtValue(), Func, Data);
}
@@ -83,20 +126,123 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) {
CtorDtorIterator(DtorsList, true));
}
-void LocalCXXRuntimeOverrides::runDestructors() {
+void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) {
+ if (CtorDtors.begin() == CtorDtors.end())
+ return;
+
+ MangleAndInterner Mangle(
+ V.getExecutionSession(),
+ (*CtorDtors.begin()).Func->getParent()->getDataLayout());
+
+ for (const auto &CtorDtor : CtorDtors) {
+ assert(CtorDtor.Func && CtorDtor.Func->hasName() &&
+ "Ctor/Dtor function must be named to be runnable under the JIT");
+
+ if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) {
+ dbgs() << " Skipping because why now?\n";
+ continue;
+ }
+
+ CtorDtorsByPriority[CtorDtor.Priority].push_back(
+ Mangle(CtorDtor.Func->getName()));
+ }
+}
+
+Error CtorDtorRunner2::run() {
+ using CtorDtorTy = void (*)();
+
+ SymbolNameSet Names;
+
+ for (auto &KV : CtorDtorsByPriority) {
+ for (auto &Name : KV.second) {
+ auto Added = Names.insert(Name).second;
+ (void)Added;
+ assert(Added && "Ctor/Dtor names clashed");
+ }
+ }
+
+ if (auto CtorDtorMap = lookup({&V}, std::move(Names))) {
+ for (auto &KV : CtorDtorsByPriority) {
+ for (auto &Name : KV.second) {
+ assert(CtorDtorMap->count(Name) && "No entry for Name");
+ auto CtorDtor = reinterpret_cast<CtorDtorTy>(
+ static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress()));
+ CtorDtor();
+ }
+ }
+ return Error::success();
+ } else
+ return CtorDtorMap.takeError();
+
+ CtorDtorsByPriority.clear();
+
+ return Error::success();
+}
+
+void LocalCXXRuntimeOverridesBase::runDestructors() {
auto& CXXDestructorDataPairs = DSOHandleOverride;
for (auto &P : CXXDestructorDataPairs)
P.first(P.second);
CXXDestructorDataPairs.clear();
}
-int LocalCXXRuntimeOverrides::CXAAtExitOverride(DestructorPtr Destructor,
- void *Arg, void *DSOHandle) {
+int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor,
+ void *Arg,
+ void *DSOHandle) {
auto& CXXDestructorDataPairs =
*reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle);
CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg));
return 0;
}
+Error LocalCXXRuntimeOverrides2::enable(VSO &V, MangleAndInterner &Mangle) {
+ SymbolMap RuntimeInterposes(
+ {{Mangle("__dso_handle"),
+ JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride),
+ JITSymbolFlags::Exported)},
+ {Mangle("__cxa_atexit"),
+ JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride),
+ JITSymbolFlags::Exported)}});
+
+ return V.define(absoluteSymbols(std::move(RuntimeInterposes)));
+}
+
+DynamicLibraryFallbackGenerator::DynamicLibraryFallbackGenerator(
+ sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow)
+ : Dylib(std::move(Dylib)), Allow(std::move(Allow)),
+ GlobalPrefix(DL.getGlobalPrefix()) {}
+
+SymbolNameSet DynamicLibraryFallbackGenerator::
+operator()(VSO &V, const SymbolNameSet &Names) {
+ orc::SymbolNameSet Added;
+ orc::SymbolMap NewSymbols;
+
+ bool HasGlobalPrefix = (GlobalPrefix != '\0');
+
+ for (auto &Name : Names) {
+ if (!Allow(Name) || (*Name).empty())
+ continue;
+
+ if (HasGlobalPrefix && (*Name).front() != GlobalPrefix)
+ continue;
+
+ std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size());
+ if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) {
+ Added.insert(Name);
+ NewSymbols[Name] = JITEvaluatedSymbol(
+ static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)),
+ JITSymbolFlags::Exported);
+ }
+ }
+
+ // Add any new symbols to V. Since the fallback generator is only called for
+ // symbols that are not already defined, this will never trigger a duplicate
+ // definition error, so we can wrap this call in a 'cantFail'.
+ if (!NewSymbols.empty())
+ cantFail(V.define(absoluteSymbols(std::move(NewSymbols))));
+
+ return Added;
+}
+
} // End namespace orc.
} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
new file mode 100644
index 000000000000..0c17f9b7ad49
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
@@ -0,0 +1,44 @@
+//===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+
+namespace llvm {
+namespace orc {
+
+IRCompileLayer2::IRCompileLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer,
+ CompileFunction Compile)
+ : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {}
+
+void IRCompileLayer2::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) {
+ std::lock_guard<std::mutex> Lock(IRLayerMutex);
+ this->NotifyCompiled = std::move(NotifyCompiled);
+}
+
+void IRCompileLayer2::emit(MaterializationResponsibility R, VModuleKey K,
+ std::unique_ptr<Module> M) {
+ assert(M && "Module must not be null");
+
+ if (auto Obj = Compile(*M)) {
+ {
+ std::lock_guard<std::mutex> Lock(IRLayerMutex);
+ if (NotifyCompiled)
+ NotifyCompiled(K, std::move(M));
+ else
+ M = nullptr;
+ }
+ BaseLayer.emit(std::move(R), std::move(K), std::move(*Obj));
+ } else {
+ R.failMaterialization();
+ getExecutionSession().reportError(Obj.takeError());
+ }
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
new file mode 100644
index 000000000000..4dd3cfdfe387
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
@@ -0,0 +1,34 @@
+//===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+namespace orc {
+
+IRTransformLayer2::IRTransformLayer2(ExecutionSession &ES,
+ IRLayer &BaseLayer,
+ TransformFunction Transform)
+ : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {}
+
+void IRTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K,
+ std::unique_ptr<Module> M) {
+ assert(M && "Module must not be null");
+
+ if (auto TransformedMod = Transform(std::move(M)))
+ BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedMod));
+ else {
+ R.failMaterialization();
+ getExecutionSession().reportError(TransformedMod.takeError());
+ }
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index 68397beae63a..9ca2c5cb4a55 100644
--- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -13,38 +13,123 @@
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/Format.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <sstream>
+using namespace llvm;
+using namespace llvm::orc;
+
+namespace {
+
+class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
+public:
+ using CompileFunction = JITCompileCallbackManager::CompileFunction;
+
+ CompileCallbackMaterializationUnit(SymbolStringPtr Name,
+ CompileFunction Compile)
+ : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}})),
+ Name(std::move(Name)), Compile(std::move(Compile)) {}
+
+private:
+ void materialize(MaterializationResponsibility R) {
+ SymbolMap Result;
+ Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
+ R.resolve(Result);
+ R.finalize();
+ }
+
+ void discard(const VSO &V, SymbolStringPtr Name) {
+ llvm_unreachable("Discard should never occur on a LMU?");
+ }
+
+ SymbolStringPtr Name;
+ CompileFunction Compile;
+};
+
+} // namespace
+
namespace llvm {
namespace orc {
void JITCompileCallbackManager::anchor() {}
void IndirectStubsManager::anchor() {}
+Expected<JITTargetAddress>
+JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
+ if (auto TrampolineAddr = getAvailableTrampolineAddr()) {
+ auto CallbackName = ES.getSymbolStringPool().intern(
+ std::string("cc") + std::to_string(++NextCallbackId));
+
+ std::lock_guard<std::mutex> Lock(CCMgrMutex);
+ AddrToSymbol[*TrampolineAddr] = CallbackName;
+ cantFail(CallbacksVSO.define(
+ llvm::make_unique<CompileCallbackMaterializationUnit>(
+ std::move(CallbackName), std::move(Compile))));
+ return *TrampolineAddr;
+ } else
+ return TrampolineAddr.takeError();
+}
+
+JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
+ JITTargetAddress TrampolineAddr) {
+ SymbolStringPtr Name;
+
+ {
+ std::unique_lock<std::mutex> Lock(CCMgrMutex);
+ auto I = AddrToSymbol.find(TrampolineAddr);
+
+ // If this address is not associated with a compile callback then report an
+ // error to the execution session and return ErrorHandlerAddress to the
+ // callee.
+ if (I == AddrToSymbol.end()) {
+ Lock.unlock();
+ std::string ErrMsg;
+ {
+ raw_string_ostream ErrMsgStream(ErrMsg);
+ ErrMsgStream << "No compile callback for trampoline at "
+ << format("0x%016x", TrampolineAddr);
+ }
+ ES.reportError(
+ make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
+ return ErrorHandlerAddress;
+ } else
+ Name = I->second;
+ }
+
+ if (auto Sym = lookup({&CallbacksVSO}, Name))
+ return Sym->getAddress();
+ else {
+ // If anything goes wrong materializing Sym then report it to the session
+ // and return the ErrorHandlerAddress;
+ ES.reportError(Sym.takeError());
+ return ErrorHandlerAddress;
+ }
+}
+
std::unique_ptr<JITCompileCallbackManager>
-createLocalCompileCallbackManager(const Triple &T,
+createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
JITTargetAddress ErrorHandlerAddress) {
switch (T.getArch()) {
default: return nullptr;
case Triple::aarch64: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
- return llvm::make_unique<CCMgrT>(ErrorHandlerAddress);
+ return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress);
}
case Triple::x86: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
- return llvm::make_unique<CCMgrT>(ErrorHandlerAddress);
+ return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress);
}
case Triple::x86_64: {
if ( T.getOS() == Triple::OSType::Win32 ) {
typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
- return llvm::make_unique<CCMgrT>(ErrorHandlerAddress);
+ return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress);
} else {
typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
- return llvm::make_unique<CCMgrT>(ErrorHandlerAddress);
+ return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress);
}
}
@@ -54,7 +139,11 @@ createLocalCompileCallbackManager(const Triple &T,
std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple &T) {
switch (T.getArch()) {
- default: return nullptr;
+ default:
+ return [](){
+ return llvm::make_unique<
+ orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
+ };
case Triple::aarch64:
return [](){
@@ -176,7 +265,6 @@ void makeAllSymbolsExternallyAccessible(Module &M) {
Function* cloneFunctionDecl(Module &Dst, const Function &F,
ValueToValueMapTy *VMap) {
- assert(F.getParent() != &Dst && "Can't copy decl over existing function.");
Function *NewF =
Function::Create(cast<FunctionType>(F.getValueType()),
F.getLinkage(), F.getName(), &Dst);
@@ -214,7 +302,6 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
ValueToValueMapTy *VMap) {
- assert(GV.getParent() != &Dst && "Can't copy decl over existing global var.");
GlobalVariable *NewGV = new GlobalVariable(
Dst, GV.getValueType(), GV.isConstant(),
GV.getLinkage(), nullptr, GV.getName(), nullptr,
@@ -236,8 +323,8 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
assert(VMap[&OrigGV] == NewGV &&
"Incorrect global variable mapping in VMap.");
assert(NewGV->getParent() != OrigGV.getParent() &&
- "moveGlobalVariable should only be used to move initializers between "
- "modules");
+ "moveGlobalVariableInitializer should only be used to move "
+ "initializers between modules");
NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
nullptr, Materializer));
diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp
new file mode 100644
index 000000000000..52ff4efe56b2
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -0,0 +1,134 @@
+//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/OrcError.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/IR/Mangler.h"
+
+namespace llvm {
+namespace orc {
+
+Expected<std::unique_ptr<LLJIT>>
+LLJIT::Create(std::unique_ptr<ExecutionSession> ES,
+ std::unique_ptr<TargetMachine> TM, DataLayout DL) {
+ return std::unique_ptr<LLJIT>(
+ new LLJIT(std::move(ES), std::move(TM), std::move(DL)));
+}
+
+Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) {
+ auto InternedName = ES->getSymbolStringPool().intern(Name);
+ SymbolMap Symbols({{InternedName, Sym}});
+ return Main.define(absoluteSymbols(std::move(Symbols)));
+}
+
+Error LLJIT::addIRModule(VSO &V, std::unique_ptr<Module> M) {
+ assert(M && "Can not add null module");
+
+ if (auto Err = applyDataLayout(*M))
+ return Err;
+
+ auto K = ES->allocateVModule();
+ return CompileLayer.add(V, K, std::move(M));
+}
+
+Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(VSO &V,
+ StringRef Name) {
+ return llvm::orc::lookup({&V}, ES->getSymbolStringPool().intern(Name));
+}
+
+LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES,
+ std::unique_ptr<TargetMachine> TM, DataLayout DL)
+ : ES(std::move(ES)), Main(this->ES->createVSO("main")), TM(std::move(TM)),
+ DL(std::move(DL)),
+ ObjLinkingLayer(*this->ES,
+ [this](VModuleKey K) { return getMemoryManager(K); }),
+ CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)),
+ CtorRunner(Main), DtorRunner(Main) {}
+
+std::shared_ptr<RuntimeDyld::MemoryManager>
+LLJIT::getMemoryManager(VModuleKey K) {
+ return llvm::make_unique<SectionMemoryManager>();
+}
+
+std::string LLJIT::mangle(StringRef UnmangledName) {
+ std::string MangledName;
+ {
+ raw_string_ostream MangledNameStream(MangledName);
+ Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
+ }
+ return MangledName;
+}
+
+Error LLJIT::applyDataLayout(Module &M) {
+ if (M.getDataLayout().isDefault())
+ M.setDataLayout(DL);
+
+ if (M.getDataLayout() != DL)
+ return make_error<StringError>(
+ "Added modules have incompatible data layouts",
+ inconvertibleErrorCode());
+
+ return Error::success();
+}
+
+void LLJIT::recordCtorDtors(Module &M) {
+ CtorRunner.add(getConstructors(M));
+ DtorRunner.add(getDestructors(M));
+}
+
+Expected<std::unique_ptr<LLLazyJIT>>
+LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES,
+ std::unique_ptr<TargetMachine> TM, DataLayout DL,
+ LLVMContext &Ctx) {
+ const Triple &TT = TM->getTargetTriple();
+
+ auto CCMgr = createLocalCompileCallbackManager(TT, *ES, 0);
+ if (!CCMgr)
+ return make_error<StringError>(
+ std::string("No callback manager available for ") + TT.str(),
+ inconvertibleErrorCode());
+
+ auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT);
+ if (!ISMBuilder)
+ return make_error<StringError>(
+ std::string("No indirect stubs manager builder for ") + TT.str(),
+ inconvertibleErrorCode());
+
+ return std::unique_ptr<LLLazyJIT>(
+ new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL), Ctx,
+ std::move(CCMgr), std::move(ISMBuilder)));
+}
+
+Error LLLazyJIT::addLazyIRModule(VSO &V, std::unique_ptr<Module> M) {
+ assert(M && "Can not add null module");
+
+ if (auto Err = applyDataLayout(*M))
+ return Err;
+
+ makeAllSymbolsExternallyAccessible(*M);
+
+ recordCtorDtors(*M);
+
+ auto K = ES->allocateVModule();
+ return CODLayer.add(V, K, std::move(M));
+}
+
+LLLazyJIT::LLLazyJIT(
+ std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM,
+ DataLayout DL, LLVMContext &Ctx,
+ std::unique_ptr<JITCompileCallbackManager> CCMgr,
+ std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder)
+ : LLJIT(std::move(ES), std::move(TM), std::move(DL)),
+ CCMgr(std::move(CCMgr)), TransformLayer(*this->ES, CompileLayer),
+ CODLayer(*this->ES, TransformLayer, *this->CCMgr, std::move(ISMBuilder),
+ [&]() -> LLVMContext & { return Ctx; }) {}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/LLVMBuild.txt b/lib/ExecutionEngine/Orc/LLVMBuild.txt
index 8f05172e77a4..3f8fe9545fcb 100644
--- a/lib/ExecutionEngine/Orc/LLVMBuild.txt
+++ b/lib/ExecutionEngine/Orc/LLVMBuild.txt
@@ -1,4 +1,4 @@
-;===- ./lib/ExecutionEngine/MCJIT/LLVMBuild.txt ----------------*- Conf -*--===;
+;===- ./lib/ExecutionEngine/Orc/LLVMBuild.txt ----------------*- Conf -*--===;
;
; The LLVM Compiler Infrastructure
;
@@ -19,4 +19,4 @@
type = Library
name = OrcJIT
parent = ExecutionEngine
-required_libraries = Core ExecutionEngine Object RuntimeDyld Support TransformUtils
+required_libraries = Core ExecutionEngine Object MC RuntimeDyld Support Target TransformUtils
diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp
new file mode 100644
index 000000000000..b9da3b7fb8d5
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/Layer.cpp
@@ -0,0 +1,106 @@
+//===-------------------- Layer.cpp - Layer interfaces --------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+namespace orc {
+
+IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {}
+IRLayer::~IRLayer() {}
+
+Error IRLayer::add(VSO &V, VModuleKey K, std::unique_ptr<Module> M) {
+ return V.define(llvm::make_unique<BasicIRLayerMaterializationUnit>(
+ *this, std::move(K), std::move(M)));
+}
+
+IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES,
+ std::unique_ptr<Module> M)
+ : MaterializationUnit(SymbolFlagsMap()), M(std::move(M)) {
+
+ MangleAndInterner Mangle(ES, this->M->getDataLayout());
+ for (auto &G : this->M->global_values()) {
+ if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() &&
+ !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) {
+ auto MangledName = Mangle(G.getName());
+ SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G);
+ SymbolToDefinition[MangledName] = &G;
+ }
+ }
+}
+
+IRMaterializationUnit::IRMaterializationUnit(
+ std::unique_ptr<Module> M, SymbolFlagsMap SymbolFlags,
+ SymbolNameToDefinitionMap SymbolToDefinition)
+ : MaterializationUnit(std::move(SymbolFlags)), M(std::move(M)),
+ SymbolToDefinition(std::move(SymbolToDefinition)) {}
+
+void IRMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) {
+ auto I = SymbolToDefinition.find(Name);
+ assert(I != SymbolToDefinition.end() &&
+ "Symbol not provided by this MU, or previously discarded");
+ assert(!I->second->isDeclaration() &&
+ "Discard should only apply to definitions");
+ I->second->setLinkage(GlobalValue::AvailableExternallyLinkage);
+ SymbolToDefinition.erase(I);
+}
+
+BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit(
+ IRLayer &L, VModuleKey K, std::unique_ptr<Module> M)
+ : IRMaterializationUnit(L.getExecutionSession(), std::move(M)),
+ L(L), K(std::move(K)) {}
+
+void BasicIRLayerMaterializationUnit::materialize(
+ MaterializationResponsibility R) {
+ L.emit(std::move(R), std::move(K), std::move(M));
+}
+
+ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {}
+
+ObjectLayer::~ObjectLayer() {}
+
+Error ObjectLayer::add(VSO &V, VModuleKey K, std::unique_ptr<MemoryBuffer> O) {
+ return V.define(llvm::make_unique<BasicObjectLayerMaterializationUnit>(
+ *this, std::move(K), std::move(O)));
+}
+
+BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit(
+ ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O)
+ : MaterializationUnit(SymbolFlagsMap()), L(L), K(std::move(K)),
+ O(std::move(O)) {
+
+ auto &ES = L.getExecutionSession();
+ auto Obj = cantFail(
+ object::ObjectFile::createObjectFile(this->O->getMemBufferRef()));
+
+ for (auto &Sym : Obj->symbols()) {
+ if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) &&
+ (Sym.getFlags() & object::BasicSymbolRef::SF_Exported)) {
+ auto InternedName =
+ ES.getSymbolStringPool().intern(cantFail(Sym.getName()));
+ SymbolFlags[InternedName] = JITSymbolFlags::fromObjectSymbol(Sym);
+ }
+ }
+}
+
+void BasicObjectLayerMaterializationUnit::materialize(
+ MaterializationResponsibility R) {
+ L.emit(std::move(R), std::move(K), std::move(O));
+}
+
+void BasicObjectLayerMaterializationUnit::discard(const VSO &V,
+ SymbolStringPtr Name) {
+ // FIXME: Support object file level discard. This could be done by building a
+ // filter to pass to the object layer along with the object itself.
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp
new file mode 100644
index 000000000000..18be9a042f7f
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/Legacy.cpp
@@ -0,0 +1,68 @@
+//===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Legacy.h"
+
+namespace llvm {
+namespace orc {
+
+void SymbolResolver::anchor() {}
+
+JITSymbolResolverAdapter::JITSymbolResolverAdapter(
+ ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR)
+ : ES(ES), R(R), MR(MR) {}
+
+Expected<JITSymbolResolverAdapter::LookupResult>
+JITSymbolResolverAdapter::lookup(const LookupSet &Symbols) {
+ SymbolNameSet InternedSymbols;
+ for (auto &S : Symbols)
+ InternedSymbols.insert(ES.getSymbolStringPool().intern(S));
+
+ auto LookupFn = [&, this](std::shared_ptr<AsynchronousSymbolQuery> Q,
+ SymbolNameSet Unresolved) {
+ return R.lookup(std::move(Q), std::move(Unresolved));
+ };
+
+ auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) {
+ if (MR)
+ MR->addDependenciesForAll(Deps);
+ };
+
+ auto InternedResult =
+ ES.legacyLookup(ES, std::move(LookupFn), std::move(InternedSymbols),
+ false, RegisterDependencies);
+
+ if (!InternedResult)
+ return InternedResult.takeError();
+
+ JITSymbolResolver::LookupResult Result;
+ for (auto &KV : *InternedResult)
+ Result[*KV.first] = KV.second;
+
+ return Result;
+}
+
+Expected<JITSymbolResolverAdapter::LookupFlagsResult>
+JITSymbolResolverAdapter::lookupFlags(const LookupSet &Symbols) {
+ SymbolNameSet InternedSymbols;
+ for (auto &S : Symbols)
+ InternedSymbols.insert(ES.getSymbolStringPool().intern(S));
+
+ SymbolFlagsMap SymbolFlags = R.lookupFlags(InternedSymbols);
+ LookupFlagsResult Result;
+ for (auto &KV : SymbolFlags) {
+ ResolvedStrings.insert(KV.first);
+ Result[*KV.first] = KV.second;
+ }
+
+ return Result;
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp
index 8f2d6fd6c32b..3796e3d37bc2 100644
--- a/lib/ExecutionEngine/Orc/NullResolver.cpp
+++ b/lib/ExecutionEngine/Orc/NullResolver.cpp
@@ -14,11 +14,23 @@
namespace llvm {
namespace orc {
-JITSymbol NullResolver::findSymbol(const std::string &Name) {
+SymbolFlagsMap NullResolver::lookupFlags(const SymbolNameSet &Symbols) {
+ return SymbolFlagsMap();
+}
+
+SymbolNameSet
+NullResolver::lookup(std::shared_ptr<AsynchronousSymbolQuery> Query,
+ SymbolNameSet Symbols) {
+ assert(Symbols.empty() && "Null resolver: Symbols must be empty");
+ return Symbols;
+}
+
+JITSymbol NullLegacyResolver::findSymbol(const std::string &Name) {
llvm_unreachable("Unexpected cross-object symbol reference");
}
-JITSymbol NullResolver::findSymbolInLogicalDylib(const std::string &Name) {
+JITSymbol
+NullLegacyResolver::findSymbolInLogicalDylib(const std::string &Name) {
llvm_unreachable("Unexpected cross-object symbol reference");
}
diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
new file mode 100644
index 000000000000..6980c8140fd0
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
@@ -0,0 +1,34 @@
+//===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+namespace orc {
+
+ObjectTransformLayer2::ObjectTransformLayer2(ExecutionSession &ES,
+ ObjectLayer &BaseLayer,
+ TransformFunction Transform)
+ : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {}
+
+void ObjectTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K,
+ std::unique_ptr<MemoryBuffer> O) {
+ assert(O && "Module must not be null");
+
+ if (auto TransformedObj = Transform(std::move(O)))
+ BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedObj));
+ else {
+ R.failMaterialization();
+ getExecutionSession().reportError(TransformedObj.takeError());
+ }
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp
index f945acaf95ee..d6005d24a648 100644
--- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp
+++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp
@@ -9,28 +9,20 @@
#include "OrcCBindingsStack.h"
#include "llvm-c/OrcBindings.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
using namespace llvm;
-LLVMSharedModuleRef LLVMOrcMakeSharedModule(LLVMModuleRef Mod) {
- return wrap(new std::shared_ptr<Module>(unwrap(Mod)));
-}
-
-void LLVMOrcDisposeSharedModuleRef(LLVMSharedModuleRef SharedMod) {
- delete unwrap(SharedMod);
-}
-
LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) {
TargetMachine *TM2(unwrap(TM));
Triple T(TM2->getTargetTriple());
- auto CompileCallbackMgr = orc::createLocalCompileCallbackManager(T, 0);
auto IndirectStubsMgrBuilder =
orc::createLocalIndirectStubsManagerBuilder(T);
- OrcCBindingsStack *JITStack = new OrcCBindingsStack(
- *TM2, std::move(CompileCallbackMgr), IndirectStubsMgrBuilder);
+ OrcCBindingsStack *JITStack =
+ new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder));
return wrap(JITStack);
}
@@ -75,24 +67,24 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack,
LLVMOrcErrorCode
LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack,
- LLVMOrcModuleHandle *RetHandle,
- LLVMSharedModuleRef Mod,
+ LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod,
LLVMOrcSymbolResolverFn SymbolResolver,
void *SymbolResolverCtx) {
OrcCBindingsStack &J = *unwrap(JITStack);
- std::shared_ptr<Module> *M(unwrap(Mod));
- return J.addIRModuleEager(*RetHandle, *M, SymbolResolver, SymbolResolverCtx);
+ std::unique_ptr<Module> M(unwrap(Mod));
+ return J.addIRModuleEager(*RetHandle, std::move(M), SymbolResolver,
+ SymbolResolverCtx);
}
LLVMOrcErrorCode
LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack,
- LLVMOrcModuleHandle *RetHandle,
- LLVMSharedModuleRef Mod,
+ LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod,
LLVMOrcSymbolResolverFn SymbolResolver,
void *SymbolResolverCtx) {
OrcCBindingsStack &J = *unwrap(JITStack);
- std::shared_ptr<Module> *M(unwrap(Mod));
- return J.addIRModuleLazy(*RetHandle, *M, SymbolResolver, SymbolResolverCtx);
+ std::unique_ptr<Module> M(unwrap(Mod));
+ return J.addIRModuleLazy(*RetHandle, std::move(M), SymbolResolver,
+ SymbolResolverCtx);
}
LLVMOrcErrorCode
@@ -120,9 +112,27 @@ LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack,
return J.findSymbolAddress(*RetAddr, SymbolName, true);
}
+LLVMOrcErrorCode LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack,
+ LLVMOrcTargetAddress *RetAddr,
+ LLVMOrcModuleHandle H,
+ const char *SymbolName) {
+ OrcCBindingsStack &J = *unwrap(JITStack);
+ return J.findSymbolAddressIn(*RetAddr, H, SymbolName, true);
+}
+
LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) {
auto *J = unwrap(JITStack);
auto Err = J->shutdown();
delete J;
return Err;
}
+
+void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L)
+{
+ unwrap(JITStack)->RegisterJITEventListener(unwrap(L));
+}
+
+void LLVMOrcUnregisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L)
+{
+ unwrap(JITStack)->UnregisterJITEventListener(unwrap(L));
+}
diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
index 405970e063d8..b9f8a370d2f0 100644
--- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
+++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
@@ -15,6 +15,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
@@ -33,6 +34,7 @@
#include <algorithm>
#include <cstdint>
#include <functional>
+#include <map>
#include <memory>
#include <set>
#include <string>
@@ -42,68 +44,61 @@ namespace llvm {
class OrcCBindingsStack;
-DEFINE_SIMPLE_CONVERSION_FUNCTIONS(std::shared_ptr<Module>,
- LLVMSharedModuleRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
namespace detail {
+// FIXME: Kill this off once the Layer concept becomes an interface.
+class GenericLayer {
+public:
+ virtual ~GenericLayer() = default;
- class GenericHandle {
- public:
- virtual ~GenericHandle() = default;
-
- virtual JITSymbol findSymbolIn(const std::string &Name,
- bool ExportedSymbolsOnly) = 0;
- virtual Error removeModule() = 0;
+ virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
+ bool ExportedSymbolsOnly) = 0;
+ virtual Error removeModule(orc::VModuleKey K) = 0;
};
- template <typename LayerT> class GenericHandleImpl : public GenericHandle {
+ template <typename LayerT> class GenericLayerImpl : public GenericLayer {
public:
- GenericHandleImpl(LayerT &Layer, typename LayerT::ModuleHandleT Handle)
- : Layer(Layer), Handle(std::move(Handle)) {}
+ GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
- JITSymbol findSymbolIn(const std::string &Name,
+ JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) override {
- return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly);
+ return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
}
- Error removeModule() override { return Layer.removeModule(Handle); }
+ Error removeModule(orc::VModuleKey K) override {
+ return Layer.removeModule(K);
+ }
private:
LayerT &Layer;
- typename LayerT::ModuleHandleT Handle;
};
template <>
- class GenericHandleImpl<orc::RTDyldObjectLinkingLayer>
- : public GenericHandle {
+ class GenericLayerImpl<orc::RTDyldObjectLinkingLayer> : public GenericLayer {
private:
using LayerT = orc::RTDyldObjectLinkingLayer;
public:
+ GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
- GenericHandleImpl(LayerT &Layer, typename LayerT::ObjHandleT Handle)
- : Layer(Layer), Handle(std::move(Handle)) {}
-
- JITSymbol findSymbolIn(const std::string &Name,
+ JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) override {
- return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly);
+ return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
}
- Error removeModule() override { return Layer.removeObject(Handle); }
+ Error removeModule(orc::VModuleKey K) override {
+ return Layer.removeObject(K);
+ }
private:
LayerT &Layer;
- typename LayerT::ObjHandleT Handle;
};
-
- template <typename LayerT, typename HandleT>
- std::unique_ptr<GenericHandleImpl<LayerT>>
- createGenericHandle(LayerT &Layer, HandleT Handle) {
- return llvm::make_unique<GenericHandleImpl<LayerT>>(Layer,
- std::move(Handle));
+ template <typename LayerT>
+ std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
+ return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
}
} // end namespace detail
@@ -126,20 +121,123 @@ private:
using OwningObject = object::OwningBinary<object::ObjectFile>;
-public:
- using ModuleHandleT = unsigned;
+ class CBindingsResolver : public orc::SymbolResolver {
+ public:
+ CBindingsResolver(OrcCBindingsStack &Stack,
+ LLVMOrcSymbolResolverFn ExternalResolver,
+ void *ExternalResolverCtx)
+ : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
+ ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
+
+ orc::SymbolFlagsMap
+ lookupFlags(const orc::SymbolNameSet &Symbols) override {
+ orc::SymbolFlagsMap SymbolFlags;
+
+ for (auto &S : Symbols) {
+ if (auto Sym = findSymbol(*S))
+ SymbolFlags[S] = Sym.getFlags();
+ else if (auto Err = Sym.takeError()) {
+ Stack.reportError(std::move(Err));
+ return orc::SymbolFlagsMap();
+ }
+ }
+
+ return SymbolFlags;
+ }
+
+ orc::SymbolNameSet
+ lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
+ orc::SymbolNameSet Symbols) override {
+ orc::SymbolNameSet UnresolvedSymbols;
+
+ for (auto &S : Symbols) {
+ if (auto Sym = findSymbol(*S)) {
+ if (auto Addr = Sym.getAddress()) {
+ Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
+ Query->notifySymbolReady();
+ } else {
+ Stack.ES.legacyFailQuery(*Query, Addr.takeError());
+ return orc::SymbolNameSet();
+ }
+ } else if (auto Err = Sym.takeError()) {
+ Stack.ES.legacyFailQuery(*Query, std::move(Err));
+ return orc::SymbolNameSet();
+ } else
+ UnresolvedSymbols.insert(S);
+ }
+
+ if (Query->isFullyResolved())
+ Query->handleFullyResolved();
+
+ if (Query->isFullyReady())
+ Query->handleFullyReady();
+
+ return UnresolvedSymbols;
+ }
+
+ private:
+ JITSymbol findSymbol(const std::string &Name) {
+ // Search order:
+ // 1. JIT'd symbols.
+ // 2. Runtime overrides.
+ // 3. External resolver (if present).
+
+ if (auto Sym = Stack.CODLayer.findSymbol(Name, true))
+ return Sym;
+ else if (auto Err = Sym.takeError())
+ return Sym.takeError();
+ if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
+ return Sym;
+
+ if (ExternalResolver)
+ return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
+ JITSymbolFlags::Exported);
+
+ return JITSymbol(nullptr);
+ }
+
+ OrcCBindingsStack &Stack;
+ LLVMOrcSymbolResolverFn ExternalResolver;
+ void *ExternalResolverCtx = nullptr;
+ };
+
+public:
OrcCBindingsStack(TargetMachine &TM,
- std::unique_ptr<CompileCallbackMgr> CCMgr,
IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
- : DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()),
- CCMgr(std::move(CCMgr)),
- ObjectLayer(
- []() {
- return std::make_shared<SectionMemoryManager>();
- }),
+ : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)),
+ DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()),
+ ObjectLayer(ES,
+ [this](orc::VModuleKey K) {
+ auto ResolverI = Resolvers.find(K);
+ assert(ResolverI != Resolvers.end() &&
+ "No resolver for module K");
+ auto Resolver = std::move(ResolverI->second);
+ Resolvers.erase(ResolverI);
+ return ObjLayerT::Resources{
+ std::make_shared<SectionMemoryManager>(), Resolver};
+ },
+ nullptr,
+ [this](orc::VModuleKey K, const object::ObjectFile &Obj,
+ const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
+ this->notifyFinalized(K, Obj, LoadedObjInfo);
+ },
+ [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
+ this->notifyFreed(K, Obj);
+ }),
CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)),
- CODLayer(CompileLayer,
+ CODLayer(ES, CompileLayer,
+ [this](orc::VModuleKey K) {
+ auto ResolverI = Resolvers.find(K);
+ assert(ResolverI != Resolvers.end() &&
+ "No resolver for module K");
+ return ResolverI->second;
+ },
+ [this](orc::VModuleKey K,
+ std::shared_ptr<orc::SymbolResolver> Resolver) {
+ assert(!Resolvers.count(K) && "Resolver already present");
+ Resolvers[K] = std::move(Resolver);
+ },
[](Function &F) { return std::set<Function *>({&F}); },
*this->CCMgr, std::move(IndirectStubsMgrBuilder), false),
CXXRuntimeOverrides(
@@ -174,15 +272,15 @@ public:
createLazyCompileCallback(JITTargetAddress &RetAddr,
LLVMOrcLazyCompileCallbackFn Callback,
void *CallbackCtx) {
- if (auto CCInfoOrErr = CCMgr->getCompileCallback()) {
- auto &CCInfo = *CCInfoOrErr;
- CCInfo.setCompileAction([=]() -> JITTargetAddress {
- return Callback(wrap(this), CallbackCtx);
- });
- RetAddr = CCInfo.getAddress();
+ auto WrappedCallback = [=]() -> JITTargetAddress {
+ return Callback(wrap(this), CallbackCtx);
+ };
+
+ if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) {
+ RetAddr = *CCAddr;
return LLVMOrcErrSuccess;
} else
- return mapError(CCInfoOrErr.takeError());
+ return mapError(CCAddr.takeError());
}
LLVMOrcErrorCode createIndirectStub(StringRef StubName,
@@ -195,42 +293,9 @@ public:
JITTargetAddress Addr) {
return mapError(IndirectStubsMgr->updatePointer(Name, Addr));
}
-
- std::shared_ptr<JITSymbolResolver>
- createResolver(LLVMOrcSymbolResolverFn ExternalResolver,
- void *ExternalResolverCtx) {
- return orc::createLambdaResolver(
- [this, ExternalResolver, ExternalResolverCtx](const std::string &Name)
- -> JITSymbol {
- // Search order:
- // 1. JIT'd symbols.
- // 2. Runtime overrides.
- // 3. External resolver (if present).
-
- if (auto Sym = CODLayer.findSymbol(Name, true))
- return Sym;
- else if (auto Err = Sym.takeError())
- return Sym.takeError();
-
- if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name))
- return Sym;
-
- if (ExternalResolver)
- return JITSymbol(
- ExternalResolver(Name.c_str(), ExternalResolverCtx),
- JITSymbolFlags::Exported);
-
- return JITSymbol(nullptr);
- },
- [](const std::string &Name) -> JITSymbol {
- return JITSymbol(nullptr);
- });
- }
-
template <typename LayerT>
LLVMOrcErrorCode
- addIRModule(ModuleHandleT &RetHandle, LayerT &Layer,
- std::shared_ptr<Module> M,
+ addIRModule(orc::VModuleKey &RetKey, LayerT &Layer, std::unique_ptr<Module> M,
std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
@@ -247,79 +312,73 @@ public:
for (auto Dtor : orc::getDestructors(*M))
DtorNames.push_back(mangle(Dtor.Func->getName()));
- // Create the resolver.
- auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx);
-
// Add the module to the JIT.
- ModuleHandleT H;
- if (auto LHOrErr = Layer.addModule(std::move(M), std::move(Resolver)))
- H = createHandle(Layer, *LHOrErr);
- else
- return mapError(LHOrErr.takeError());
+ RetKey = ES.allocateVModule();
+ Resolvers[RetKey] = std::make_shared<CBindingsResolver>(
+ *this, ExternalResolver, ExternalResolverCtx);
+ if (auto Err = Layer.addModule(RetKey, std::move(M)))
+ return mapError(std::move(Err));
+
+ KeyLayers[RetKey] = detail::createGenericLayer(Layer);
// Run the static constructors, and save the static destructor runner for
// execution when the JIT is torn down.
- orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), H);
+ orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames),
+ RetKey);
if (auto Err = CtorRunner.runViaLayer(*this))
return mapError(std::move(Err));
- IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H);
+ IRStaticDestructorRunners.emplace_back(std::move(DtorNames), RetKey);
- RetHandle = H;
return LLVMOrcErrSuccess;
}
- LLVMOrcErrorCode addIRModuleEager(ModuleHandleT &RetHandle,
- std::shared_ptr<Module> M,
+ LLVMOrcErrorCode addIRModuleEager(orc::VModuleKey &RetKey,
+ std::unique_ptr<Module> M,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
- return addIRModule(RetHandle, CompileLayer, std::move(M),
+ return addIRModule(RetKey, CompileLayer, std::move(M),
llvm::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
- LLVMOrcErrorCode addIRModuleLazy(ModuleHandleT &RetHandle,
- std::shared_ptr<Module> M,
+ LLVMOrcErrorCode addIRModuleLazy(orc::VModuleKey &RetKey,
+ std::unique_ptr<Module> M,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
- return addIRModule(RetHandle, CODLayer, std::move(M),
+ return addIRModule(RetKey, CODLayer, std::move(M),
llvm::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
- LLVMOrcErrorCode removeModule(ModuleHandleT H) {
- if (auto Err = GenericHandles[H]->removeModule())
+ LLVMOrcErrorCode removeModule(orc::VModuleKey K) {
+ // FIXME: Should error release the module key?
+ if (auto Err = KeyLayers[K]->removeModule(K))
return mapError(std::move(Err));
- GenericHandles[H] = nullptr;
- FreeHandleIndexes.push_back(H);
+ ES.releaseVModule(K);
+ KeyLayers.erase(K);
return LLVMOrcErrSuccess;
}
- LLVMOrcErrorCode addObject(ModuleHandleT &RetHandle,
+ LLVMOrcErrorCode addObject(orc::VModuleKey &RetKey,
std::unique_ptr<MemoryBuffer> ObjBuffer,
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
- if (auto ObjOrErr =
- object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())) {
- auto &Obj = *ObjOrErr;
- auto OwningObj =
- std::make_shared<OwningObject>(std::move(Obj), std::move(ObjBuffer));
+ if (auto Obj = object::ObjectFile::createObjectFile(
+ ObjBuffer->getMemBufferRef())) {
- // Create the resolver.
- auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx);
+ RetKey = ES.allocateVModule();
+ Resolvers[RetKey] = std::make_shared<CBindingsResolver>(
+ *this, ExternalResolver, ExternalResolverCtx);
- ModuleHandleT H;
- if (auto HOrErr = ObjectLayer.addObject(std::move(OwningObj),
- std::move(Resolver)))
- H = createHandle(ObjectLayer, *HOrErr);
- else
- return mapError(HOrErr.takeError());
+ if (auto Err = ObjectLayer.addObject(RetKey, std::move(ObjBuffer)))
+ return mapError(std::move(Err));
- RetHandle = H;
+ KeyLayers[RetKey] = detail::createGenericLayer(ObjectLayer);
return LLVMOrcErrSuccess;
} else
- return mapError(ObjOrErr.takeError());
+ return mapError(Obj.takeError());
}
JITSymbol findSymbol(const std::string &Name,
@@ -329,9 +388,10 @@ public:
return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
}
- JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name,
+ JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
bool ExportedSymbolsOnly) {
- return GenericHandles[H]->findSymbolIn(Name, ExportedSymbolsOnly);
+ assert(KeyLayers.count(K) && "looking up symbol in unknown module");
+ return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
}
LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr,
@@ -354,26 +414,48 @@ public:
return LLVMOrcErrSuccess;
}
+ LLVMOrcErrorCode findSymbolAddressIn(JITTargetAddress &RetAddr,
+ orc::VModuleKey K,
+ const std::string &Name,
+ bool ExportedSymbolsOnly) {
+ RetAddr = 0;
+ if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
+ // Successful lookup, non-null symbol:
+ if (auto AddrOrErr = Sym.getAddress()) {
+ RetAddr = *AddrOrErr;
+ return LLVMOrcErrSuccess;
+ } else
+ return mapError(AddrOrErr.takeError());
+ } else if (auto Err = Sym.takeError()) {
+ // Lookup failure - report error.
+ return mapError(std::move(Err));
+ }
+ // Otherwise we had a successful lookup but got a null result. We already
+ // set RetAddr to '0' above, so just return success.
+ return LLVMOrcErrSuccess;
+ }
+
const std::string &getErrorMessage() const { return ErrMsg; }
-private:
- template <typename LayerT, typename HandleT>
- unsigned createHandle(LayerT &Layer, HandleT Handle) {
- unsigned NewHandle;
- if (!FreeHandleIndexes.empty()) {
- NewHandle = FreeHandleIndexes.back();
- FreeHandleIndexes.pop_back();
- GenericHandles[NewHandle] =
- detail::createGenericHandle(Layer, std::move(Handle));
- return NewHandle;
- } else {
- NewHandle = GenericHandles.size();
- GenericHandles.push_back(
- detail::createGenericHandle(Layer, std::move(Handle)));
+ void RegisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+ EventListeners.push_back(L);
+ }
+
+ void UnregisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+
+ auto I = find(reverse(EventListeners), L);
+ if (I != EventListeners.rend()) {
+ std::swap(*I, EventListeners.back());
+ EventListeners.pop_back();
}
- return NewHandle;
}
+private:
+
LLVMOrcErrorCode mapError(Error Err) {
LLVMOrcErrorCode Result = LLVMOrcErrSuccess;
handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) {
@@ -386,22 +468,44 @@ private:
return Result;
}
+ void reportError(Error Err) {
+ // FIXME: Report errors on the execution session.
+ logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
+ };
+
+ void notifyFinalized(orc::VModuleKey K,
+ const object::ObjectFile &Obj,
+ const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
+ for (auto &Listener : EventListeners)
+ Listener->NotifyObjectEmitted(Obj, LoadedObjInfo);
+ }
+
+ void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
+ for (auto &Listener : EventListeners)
+ Listener->NotifyFreeingObject(Obj);
+ }
+
+ orc::ExecutionSession ES;
+ std::unique_ptr<CompileCallbackMgr> CCMgr;
+
+ std::vector<JITEventListener *> EventListeners;
+
DataLayout DL;
SectionMemoryManager CCMgrMemMgr;
std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
- std::unique_ptr<CompileCallbackMgr> CCMgr;
ObjLayerT ObjectLayer;
CompileLayerT CompileLayer;
CODLayerT CODLayer;
- std::vector<std::unique_ptr<detail::GenericHandle>> GenericHandles;
- std::vector<unsigned> FreeHandleIndexes;
+ std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides;
std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
std::string ErrMsg;
+
+ std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers;
};
} // end namespace llvm
diff --git a/lib/ExecutionEngine/Orc/OrcError.cpp b/lib/ExecutionEngine/Orc/OrcError.cpp
index c218cb9a523c..f4102b359a6b 100644
--- a/lib/ExecutionEngine/Orc/OrcError.cpp
+++ b/lib/ExecutionEngine/Orc/OrcError.cpp
@@ -29,6 +29,12 @@ public:
std::string message(int condition) const override {
switch (static_cast<OrcErrorCode>(condition)) {
+ case OrcErrorCode::UnknownORCError:
+ return "Unknown ORC error";
+ case OrcErrorCode::DuplicateDefinition:
+ return "Duplicate symbol definition";
+ case OrcErrorCode::JITSymbolNotFound:
+ return "JIT symbol not found";
case OrcErrorCode::RemoteAllocatorDoesNotExist:
return "Remote allocator does not exist";
case OrcErrorCode::RemoteAllocatorIdAlreadyInUse:
@@ -45,8 +51,6 @@ public:
return "Could not negotiate RPC function";
case OrcErrorCode::RPCResponseAbandoned:
return "RPC response abandoned";
- case OrcErrorCode::JITSymbolNotFound:
- return "JIT symbol not found";
case OrcErrorCode::UnexpectedRPCCall:
return "Unexpected RPC call";
case OrcErrorCode::UnexpectedRPCResponse:
@@ -67,6 +71,7 @@ static ManagedStatic<OrcErrorCategory> OrcErrCat;
namespace llvm {
namespace orc {
+char DuplicateDefinition::ID = 0;
char JITSymbolNotFound::ID = 0;
std::error_code orcError(OrcErrorCode ErrCode) {
@@ -74,6 +79,22 @@ std::error_code orcError(OrcErrorCode ErrCode) {
return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat);
}
+
+DuplicateDefinition::DuplicateDefinition(std::string SymbolName)
+ : SymbolName(std::move(SymbolName)) {}
+
+std::error_code DuplicateDefinition::convertToErrorCode() const {
+ return orcError(OrcErrorCode::DuplicateDefinition);
+}
+
+void DuplicateDefinition::log(raw_ostream &OS) const {
+ OS << "Duplicate definition of symbol '" << SymbolName << "'";
+}
+
+const std::string &DuplicateDefinition::getSymbolName() const {
+ return SymbolName;
+}
+
JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName)
: SymbolName(std::move(SymbolName)) {}
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
index f89f21adff41..4def579e7097 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
@@ -125,8 +125,13 @@ OrcMCJITReplacement::runFunction(Function *F,
}
void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) {
- for (auto &M : LocalModules)
- ExecutionEngine::runStaticConstructorsDestructors(*M, isDtors);
+ auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors;
+
+ for (auto &KV : CtorDtorsMap)
+ cantFail(CtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first)
+ .runViaLayer(LazyEmitLayer));
+
+ CtorDtorsMap.clear();
}
} // End namespace orc.
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
index 1dc8d4ac7bc5..abe89ce70af9 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
@@ -21,6 +21,7 @@
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/JITSymbol.h"
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
@@ -54,6 +55,7 @@ class ObjectCache;
namespace orc {
class OrcMCJITReplacement : public ExecutionEngine {
+
// OrcMCJITReplacement needs to do a little extra book-keeping to ensure that
// Orc's automatic finalization doesn't kick in earlier than MCJIT clients are
// expecting - see finalizeMemory.
@@ -138,18 +140,75 @@ class OrcMCJITReplacement : public ExecutionEngine {
std::shared_ptr<MCJITMemoryManager> ClientMM;
};
- class LinkingResolver : public JITSymbolResolver {
+ class LinkingORCResolver : public orc::SymbolResolver {
public:
- LinkingResolver(OrcMCJITReplacement &M) : M(M) {}
+ LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {}
+
+ SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) override {
+ SymbolFlagsMap SymbolFlags;
+
+ for (auto &S : Symbols) {
+ if (auto Sym = M.findMangledSymbol(*S)) {
+ SymbolFlags[S] = Sym.getFlags();
+ } else if (auto Err = Sym.takeError()) {
+ M.reportError(std::move(Err));
+ return SymbolFlagsMap();
+ } else {
+ if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) {
+ SymbolFlags[S] = Sym2.getFlags();
+ } else if (auto Err = Sym2.takeError()) {
+ M.reportError(std::move(Err));
+ return SymbolFlagsMap();
+ }
+ }
+ }
- JITSymbol findSymbol(const std::string &Name) override {
- return M.ClientResolver->findSymbol(Name);
+ return SymbolFlags;
}
- JITSymbol findSymbolInLogicalDylib(const std::string &Name) override {
- if (auto Sym = M.findMangledSymbol(Name))
- return Sym;
- return M.ClientResolver->findSymbolInLogicalDylib(Name);
+ SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query,
+ SymbolNameSet Symbols) override {
+ SymbolNameSet UnresolvedSymbols;
+ bool NewSymbolsResolved = false;
+
+ for (auto &S : Symbols) {
+ if (auto Sym = M.findMangledSymbol(*S)) {
+ if (auto Addr = Sym.getAddress()) {
+ Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
+ Query->notifySymbolReady();
+ NewSymbolsResolved = true;
+ } else {
+ M.ES.legacyFailQuery(*Query, Addr.takeError());
+ return SymbolNameSet();
+ }
+ } else if (auto Err = Sym.takeError()) {
+ M.ES.legacyFailQuery(*Query, std::move(Err));
+ return SymbolNameSet();
+ } else {
+ if (auto Sym2 = M.ClientResolver->findSymbol(*S)) {
+ if (auto Addr = Sym2.getAddress()) {
+ Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags()));
+ Query->notifySymbolReady();
+ NewSymbolsResolved = true;
+ } else {
+ M.ES.legacyFailQuery(*Query, Addr.takeError());
+ return SymbolNameSet();
+ }
+ } else if (auto Err = Sym2.takeError()) {
+ M.ES.legacyFailQuery(*Query, std::move(Err));
+ return SymbolNameSet();
+ } else
+ UnresolvedSymbols.insert(S);
+ }
+ }
+
+ if (NewSymbolsResolved && Query->isFullyResolved())
+ Query->handleFullyResolved();
+
+ if (NewSymbolsResolved && Query->isFullyReady())
+ Query->handleFullyReady();
+
+ return UnresolvedSymbols;
}
private:
@@ -160,26 +219,37 @@ private:
static ExecutionEngine *
createOrcMCJITReplacement(std::string *ErrorMsg,
std::shared_ptr<MCJITMemoryManager> MemMgr,
- std::shared_ptr<JITSymbolResolver> Resolver,
+ std::shared_ptr<LegacyJITSymbolResolver> Resolver,
std::unique_ptr<TargetMachine> TM) {
return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver),
std::move(TM));
}
+ void reportError(Error Err) {
+ logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: ");
+ }
+
public:
- OrcMCJITReplacement(
- std::shared_ptr<MCJITMemoryManager> MemMgr,
- std::shared_ptr<JITSymbolResolver> ClientResolver,
- std::unique_ptr<TargetMachine> TM)
- : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)),
- MemMgr(std::make_shared<MCJITReplacementMemMgr>(*this,
- std::move(MemMgr))),
- Resolver(std::make_shared<LinkingResolver>(*this)),
+ OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr,
+ std::shared_ptr<LegacyJITSymbolResolver> ClientResolver,
+ std::unique_ptr<TargetMachine> TM)
+ : ExecutionEngine(TM->createDataLayout()),
+ TM(std::move(TM)),
+ MemMgr(
+ std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))),
+ Resolver(std::make_shared<LinkingORCResolver>(*this)),
ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this),
NotifyFinalized(*this),
- ObjectLayer([this]() { return this->MemMgr; }, NotifyObjectLoaded,
- NotifyFinalized),
- CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)),
+ ObjectLayer(
+ ES,
+ [this](VModuleKey K) {
+ return ObjectLayerT::Resources{this->MemMgr, this->Resolver};
+ },
+ NotifyObjectLoaded, NotifyFinalized),
+ CompileLayer(ObjectLayer, SimpleCompiler(*this->TM),
+ [this](VModuleKey K, std::unique_ptr<Module> M) {
+ Modules.push_back(std::move(M));
+ }),
LazyEmitLayer(CompileLayer) {}
static void Register() {
@@ -194,43 +264,63 @@ public:
} else {
assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch");
}
- auto *MPtr = M.release();
- ShouldDelete[MPtr] = true;
- auto Deleter = [this](Module *Mod) {
- auto I = ShouldDelete.find(Mod);
- if (I != ShouldDelete.end() && I->second)
- delete Mod;
- };
- LocalModules.push_back(std::shared_ptr<Module>(MPtr, std::move(Deleter)));
- cantFail(LazyEmitLayer.addModule(LocalModules.back(), Resolver));
+
+ // Rename, bump linkage and record static constructors and destructors.
+ // We have to do this before we hand over ownership of the module to the
+ // JIT.
+ std::vector<std::string> CtorNames, DtorNames;
+ {
+ unsigned CtorId = 0, DtorId = 0;
+ for (auto Ctor : orc::getConstructors(*M)) {
+ std::string NewCtorName = ("$static_ctor." + Twine(CtorId++)).str();
+ Ctor.Func->setName(NewCtorName);
+ Ctor.Func->setLinkage(GlobalValue::ExternalLinkage);
+ Ctor.Func->setVisibility(GlobalValue::HiddenVisibility);
+ CtorNames.push_back(mangle(NewCtorName));
+ }
+ for (auto Dtor : orc::getDestructors(*M)) {
+ std::string NewDtorName = ("$static_dtor." + Twine(DtorId++)).str();
+ dbgs() << "Found dtor: " << NewDtorName << "\n";
+ Dtor.Func->setName(NewDtorName);
+ Dtor.Func->setLinkage(GlobalValue::ExternalLinkage);
+ Dtor.Func->setVisibility(GlobalValue::HiddenVisibility);
+ DtorNames.push_back(mangle(NewDtorName));
+ }
+ }
+
+ auto K = ES.allocateVModule();
+
+ UnexecutedConstructors[K] = std::move(CtorNames);
+ UnexecutedDestructors[K] = std::move(DtorNames);
+
+ cantFail(LazyEmitLayer.addModule(K, std::move(M)));
}
void addObjectFile(std::unique_ptr<object::ObjectFile> O) override {
- auto Obj =
- std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O),
- nullptr);
- cantFail(ObjectLayer.addObject(std::move(Obj), Resolver));
+ cantFail(ObjectLayer.addObject(
+ ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData())));
}
void addObjectFile(object::OwningBinary<object::ObjectFile> O) override {
- auto Obj =
- std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O));
- cantFail(ObjectLayer.addObject(std::move(Obj), Resolver));
+ std::unique_ptr<object::ObjectFile> Obj;
+ std::unique_ptr<MemoryBuffer> ObjBuffer;
+ std::tie(Obj, ObjBuffer) = O.takeBinary();
+ cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer)));
}
void addArchive(object::OwningBinary<object::Archive> A) override {
Archives.push_back(std::move(A));
}
-
+
bool removeModule(Module *M) override {
- for (auto I = LocalModules.begin(), E = LocalModules.end(); I != E; ++I) {
- if (I->get() == M) {
- ShouldDelete[M] = false;
- LocalModules.erase(I);
- return true;
- }
- }
- return false;
+ auto I = Modules.begin();
+ for (auto E = Modules.end(); I != E; ++I)
+ if (I->get() == M)
+ break;
+ if (I == Modules.end())
+ return false;
+ Modules.erase(I);
+ return true;
}
uint64_t getSymbolAddress(StringRef Name) {
@@ -238,7 +328,7 @@ public:
}
JITSymbol findSymbol(StringRef Name) {
- return findMangledSymbol(Mangle(Name));
+ return findMangledSymbol(mangle(Name));
}
void finalizeObject() override {
@@ -318,12 +408,9 @@ private:
}
std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get();
if (ChildBin->isObject()) {
- std::unique_ptr<object::ObjectFile> ChildObj(
- static_cast<object::ObjectFile*>(ChildBinOrErr->release()));
- auto Obj =
- std::make_shared<object::OwningBinary<object::ObjectFile>>(
- std::move(ChildObj), nullptr);
- cantFail(ObjectLayer.addObject(std::move(Obj), Resolver));
+ cantFail(ObjectLayer.addObject(
+ ES.allocateVModule(),
+ MemoryBuffer::getMemBufferCopy(ChildBin->getData())));
if (auto Sym = ObjectLayer.findSymbol(Name, true))
return Sym;
}
@@ -339,12 +426,11 @@ private:
NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {}
- void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H,
- const RTDyldObjectLinkingLayer::ObjectPtr &Obj,
+ void operator()(VModuleKey K, const object::ObjectFile &Obj,
const RuntimeDyld::LoadedObjectInfo &Info) const {
- M.UnfinalizedSections[H] = std::move(M.SectionsAllocatedSinceLastLoad);
+ M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad);
M.SectionsAllocatedSinceLastLoad = SectionAddrSet();
- M.MemMgr->notifyObjectLoaded(&M, *Obj->getBinary());
+ M.MemMgr->notifyObjectLoaded(&M, Obj);
}
private:
OrcMCJITReplacement &M;
@@ -354,15 +440,16 @@ private:
public:
NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {}
- void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H) {
- M.UnfinalizedSections.erase(H);
+ void operator()(VModuleKey K, const object::ObjectFile &Obj,
+ const RuntimeDyld::LoadedObjectInfo &Info) {
+ M.UnfinalizedSections.erase(K);
}
private:
OrcMCJITReplacement &M;
};
- std::string Mangle(StringRef Name) {
+ std::string mangle(StringRef Name) {
std::string MangledName;
{
raw_string_ostream MangledNameStream(MangledName);
@@ -375,17 +462,18 @@ private:
using CompileLayerT = IRCompileLayer<ObjectLayerT, orc::SimpleCompiler>;
using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>;
+ ExecutionSession ES;
+
std::unique_ptr<TargetMachine> TM;
std::shared_ptr<MCJITReplacementMemMgr> MemMgr;
- std::shared_ptr<LinkingResolver> Resolver;
- std::shared_ptr<JITSymbolResolver> ClientResolver;
+ std::shared_ptr<LinkingORCResolver> Resolver;
+ std::shared_ptr<LegacyJITSymbolResolver> ClientResolver;
Mangler Mang;
// IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr
// delete blocks in LocalModules refer to the ShouldDelete map, so
// LocalModules needs to be destructed before ShouldDelete.
std::map<Module*, bool> ShouldDelete;
- std::vector<std::shared_ptr<Module>> LocalModules;
NotifyObjectLoadedT NotifyObjectLoaded;
NotifyFinalizedT NotifyFinalized;
@@ -394,19 +482,15 @@ private:
CompileLayerT CompileLayer;
LazyEmitLayerT LazyEmitLayer;
+ std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors;
+ std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors;
+
// We need to store ObjLayerT::ObjSetHandles for each of the object sets
// that have been emitted but not yet finalized so that we can forward the
// mapSectionAddress calls appropriately.
using SectionAddrSet = std::set<const void *>;
- struct ObjHandleCompare {
- bool operator()(ObjectLayerT::ObjHandleT H1,
- ObjectLayerT::ObjHandleT H2) const {
- return &*H1 < &*H2;
- }
- };
SectionAddrSet SectionsAllocatedSinceLastLoad;
- std::map<ObjectLayerT::ObjHandleT, SectionAddrSet, ObjHandleCompare>
- UnfinalizedSections;
+ std::map<VModuleKey, SectionAddrSet> UnfinalizedSections;
std::vector<object::OwningBinary<object::Archive>> Archives;
};
diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
new file mode 100644
index 000000000000..71b4b73ca6d3
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -0,0 +1,177 @@
+//===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+
+namespace {
+
+using namespace llvm;
+using namespace llvm::orc;
+
+class VSOSearchOrderResolver : public JITSymbolResolver {
+public:
+ VSOSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {}
+
+ Expected<LookupResult> lookup(const LookupSet &Symbols) {
+ auto &ES = MR.getTargetVSO().getExecutionSession();
+ SymbolNameSet InternedSymbols;
+
+ for (auto &S : Symbols)
+ InternedSymbols.insert(ES.getSymbolStringPool().intern(S));
+
+ auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) {
+ MR.addDependenciesForAll(Deps);
+ };
+
+ auto InternedResult =
+ MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) {
+ return ES.lookup(VSOs, InternedSymbols, RegisterDependencies, false);
+ });
+
+ if (!InternedResult)
+ return InternedResult.takeError();
+
+ LookupResult Result;
+ for (auto &KV : *InternedResult)
+ Result[*KV.first] = std::move(KV.second);
+
+ return Result;
+ }
+
+ Expected<LookupFlagsResult> lookupFlags(const LookupSet &Symbols) {
+ auto &ES = MR.getTargetVSO().getExecutionSession();
+
+ SymbolNameSet InternedSymbols;
+
+ for (auto &S : Symbols)
+ InternedSymbols.insert(ES.getSymbolStringPool().intern(S));
+
+ SymbolFlagsMap InternedResult;
+ MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) {
+ // An empty search order is pathalogical, but allowed.
+ if (VSOs.empty())
+ return;
+
+ assert(VSOs.front() && "VSOList entry can not be null");
+ InternedResult = VSOs.front()->lookupFlags(InternedSymbols);
+ });
+
+ LookupFlagsResult Result;
+ for (auto &KV : InternedResult)
+ Result[*KV.first] = std::move(KV.second);
+
+ return Result;
+ }
+
+private:
+ MaterializationResponsibility &MR;
+};
+
+} // end anonymous namespace
+
+namespace llvm {
+namespace orc {
+
+RTDyldObjectLinkingLayer2::RTDyldObjectLinkingLayer2(
+ ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager,
+ NotifyLoadedFunction NotifyLoaded, NotifyFinalizedFunction NotifyFinalized)
+ : ObjectLayer(ES), GetMemoryManager(GetMemoryManager),
+ NotifyLoaded(std::move(NotifyLoaded)),
+ NotifyFinalized(std::move(NotifyFinalized)), ProcessAllSections(false) {}
+
+void RTDyldObjectLinkingLayer2::emit(MaterializationResponsibility R,
+ VModuleKey K,
+ std::unique_ptr<MemoryBuffer> O) {
+ assert(O && "Object must not be null");
+
+ auto &ES = getExecutionSession();
+
+ auto ObjFile = object::ObjectFile::createObjectFile(*O);
+ if (!ObjFile) {
+ getExecutionSession().reportError(ObjFile.takeError());
+ R.failMaterialization();
+ }
+
+ auto MemoryManager = GetMemoryManager(K);
+
+ VSOSearchOrderResolver Resolver(R);
+ auto RTDyld = llvm::make_unique<RuntimeDyld>(*MemoryManager, Resolver);
+ RTDyld->setProcessAllSections(ProcessAllSections);
+
+ {
+ std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+
+ assert(!ActiveRTDylds.count(K) &&
+ "An active RTDyld already exists for this key?");
+ ActiveRTDylds[K] = RTDyld.get();
+
+ assert(!MemMgrs.count(K) &&
+ "A memory manager already exists for this key?");
+ MemMgrs[K] = std::move(MemoryManager);
+ }
+
+ auto Info = RTDyld->loadObject(**ObjFile);
+
+ {
+ std::set<StringRef> InternalSymbols;
+ for (auto &Sym : (*ObjFile)->symbols()) {
+ if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) {
+ if (auto SymName = Sym.getName())
+ InternalSymbols.insert(*SymName);
+ else {
+ ES.reportError(SymName.takeError());
+ R.failMaterialization();
+ return;
+ }
+ }
+ }
+
+ SymbolMap Symbols;
+ for (auto &KV : RTDyld->getSymbolTable())
+ if (!InternalSymbols.count(KV.first))
+ Symbols[ES.getSymbolStringPool().intern(KV.first)] = KV.second;
+
+ R.resolve(Symbols);
+ }
+
+ if (NotifyLoaded)
+ NotifyLoaded(K, **ObjFile, *Info);
+
+ RTDyld->finalizeWithMemoryManagerLocking();
+
+ {
+ std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+ ActiveRTDylds.erase(K);
+ }
+
+ if (RTDyld->hasError()) {
+ ES.reportError(make_error<StringError>(RTDyld->getErrorString(),
+ inconvertibleErrorCode()));
+ R.failMaterialization();
+ return;
+ }
+
+ R.finalize();
+
+ if (NotifyFinalized)
+ NotifyFinalized(K);
+}
+
+void RTDyldObjectLinkingLayer2::mapSectionAddress(
+ VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) const {
+ std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+ auto ActiveRTDyldItr = ActiveRTDylds.find(K);
+
+ assert(ActiveRTDyldItr != ActiveRTDylds.end() &&
+ "No active RTDyld instance found for key");
+ ActiveRTDyldItr->second->mapSectionAddress(LocalAddress, TargetAddr);
+}
+
+} // End namespace orc.
+} // End namespace llvm.