diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc')
24 files changed, 7037 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp new file mode 100644 index 000000000000..99bf53bc3afa --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -0,0 +1,302 @@ +//===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" + +using namespace llvm; +using namespace llvm::orc; + +static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM, + StringRef Suffix, + GVPredicate ShouldExtract) { + + auto DeleteExtractedDefs = [](GlobalValue &GV) { + // Bump the linkage: this global will be provided by the external module. + GV.setLinkage(GlobalValue::ExternalLinkage); + + // Delete the definition in the source module. + if (isa<Function>(GV)) { + auto &F = cast<Function>(GV); + F.deleteBody(); + F.setPersonalityFn(nullptr); + } else if (isa<GlobalVariable>(GV)) { + cast<GlobalVariable>(GV).setInitializer(nullptr); + } else if (isa<GlobalAlias>(GV)) { + // We need to turn deleted aliases into function or variable decls based + // on the type of their aliasee. + auto &A = cast<GlobalAlias>(GV); + Constant *Aliasee = A.getAliasee(); + assert(A.hasName() && "Anonymous alias?"); + assert(Aliasee->hasName() && "Anonymous aliasee"); + std::string AliasName = A.getName(); + + if (isa<Function>(Aliasee)) { + auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee)); + A.replaceAllUsesWith(F); + A.eraseFromParent(); + F->setName(AliasName); + } else if (isa<GlobalVariable>(Aliasee)) { + auto *G = cloneGlobalVariableDecl(*A.getParent(), + *cast<GlobalVariable>(Aliasee)); + A.replaceAllUsesWith(G); + A.eraseFromParent(); + G->setName(AliasName); + } else + llvm_unreachable("Alias to unsupported type"); + } else + llvm_unreachable("Unsupported global type"); + }; + + auto NewTSMod = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs); + auto &M = *NewTSMod.getModule(); + M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str()); + + return NewTSMod; +} + +namespace llvm { +namespace orc { + +class PartitioningIRMaterializationUnit : public IRMaterializationUnit { +public: + PartitioningIRMaterializationUnit(ExecutionSession &ES, ThreadSafeModule TSM, + VModuleKey K, CompileOnDemandLayer &Parent) + : IRMaterializationUnit(ES, std::move(TSM), std::move(K)), + Parent(Parent) {} + + PartitioningIRMaterializationUnit( + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition, + CompileOnDemandLayer &Parent) + : IRMaterializationUnit(std::move(TSM), std::move(K), + std::move(SymbolFlags), + std::move(SymbolToDefinition)), + Parent(Parent) {} + +private: + void materialize(MaterializationResponsibility R) override { + Parent.emitPartition(std::move(R), std::move(TSM), + std::move(SymbolToDefinition)); + } + + void discard(const JITDylib &V, const SymbolStringPtr &Name) override { + // All original symbols were materialized by the CODLayer and should be + // final. The function bodies provided by M should never be overridden. + llvm_unreachable("Discard should never be called on an " + "ExtractingIRMaterializationUnit"); + } + + mutable std::mutex SourceModuleMutex; + CompileOnDemandLayer &Parent; +}; + +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileRequested(GlobalValueSet Requested) { + return std::move(Requested); +} + +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) { + return None; +} + +CompileOnDemandLayer::CompileOnDemandLayer( + ExecutionSession &ES, IRLayer &BaseLayer, LazyCallThroughManager &LCTMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager) + : IRLayer(ES), BaseLayer(BaseLayer), LCTMgr(LCTMgr), + BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)) {} + +void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { + this->Partition = std::move(Partition); +} + +void CompileOnDemandLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Null module"); + + auto &ES = getExecutionSession(); + auto &M = *TSM.getModule(); + + // First, do some cleanup on the module: + cleanUpModule(M); + + // Now sort the callables and non-callables, build re-exports and lodge the + // actual module with the implementation dylib. + auto &PDR = getPerDylibResources(R.getTargetJITDylib()); + + MangleAndInterner Mangle(ES, M.getDataLayout()); + SymbolAliasMap NonCallables; + SymbolAliasMap Callables; + for (auto &GV : M.global_values()) { + if (GV.isDeclaration() || GV.hasLocalLinkage() || GV.hasAppendingLinkage()) + continue; + + auto Name = Mangle(GV.getName()); + auto Flags = JITSymbolFlags::fromGlobalValue(GV); + if (Flags.isCallable()) + Callables[Name] = SymbolAliasMapEntry(Name, Flags); + else + NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); + } + + // Create a partitioning materialization unit and lodge it with the + // implementation dylib. + if (auto Err = PDR.getImplDylib().define( + llvm::make_unique<PartitioningIRMaterializationUnit>( + ES, std::move(TSM), R.getVModuleKey(), *this))) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), true)); + R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), + std::move(Callables))); +} + +CompileOnDemandLayer::PerDylibResources & +CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { + auto I = DylibResources.find(&TargetD); + if (I == DylibResources.end()) { + auto &ImplD = getExecutionSession().createJITDylib( + TargetD.getName() + ".impl", false); + TargetD.withSearchOrderDo([&](const JITDylibSearchList &TargetSearchOrder) { + auto NewSearchOrder = TargetSearchOrder; + assert(!NewSearchOrder.empty() && + NewSearchOrder.front().first == &TargetD && + NewSearchOrder.front().second == true && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewSearchOrder.insert(std::next(NewSearchOrder.begin()), {&ImplD, true}); + ImplD.setSearchOrder(std::move(NewSearchOrder), false); + }); + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); + I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; + } + + return I->second; +} + +void CompileOnDemandLayer::cleanUpModule(Module &M) { + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + + if (F.hasAvailableExternallyLinkage()) { + F.deleteBody(); + F.setPersonalityFn(nullptr); + continue; + } + } +} + +void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { + // Expands the partition to ensure the following rules hold: + // (1) If any alias is in the partition, its aliasee is also in the partition. + // (2) If any aliasee is in the partition, its aliases are also in the + // partiton. + // (3) If any global variable is in the partition then all global variables + // are in the partition. + assert(!Partition.empty() && "Unexpected empty partition"); + + const Module &M = *(*Partition.begin())->getParent(); + bool ContainsGlobalVariables = false; + std::vector<const GlobalValue *> GVsToAdd; + + for (auto *GV : Partition) + if (isa<GlobalAlias>(GV)) + GVsToAdd.push_back( + cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee())); + else if (isa<GlobalVariable>(GV)) + ContainsGlobalVariables = true; + + for (auto &A : M.aliases()) + if (Partition.count(cast<GlobalValue>(A.getAliasee()))) + GVsToAdd.push_back(&A); + + if (ContainsGlobalVariables) + for (auto &G : M.globals()) + GVsToAdd.push_back(&G); + + for (auto *GV : GVsToAdd) + Partition.insert(GV); +} + +void CompileOnDemandLayer::emitPartition( + MaterializationResponsibility R, ThreadSafeModule TSM, + IRMaterializationUnit::SymbolNameToDefinitionMap Defs) { + + // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the + // extracted module key, extracted module, and source module key + // together. This could be used, for example, to provide a specific + // memory manager instance to the linking layer. + + auto &ES = getExecutionSession(); + + GlobalValueSet RequestedGVs; + for (auto &Name : R.getRequestedSymbols()) { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); + } + + auto GVsToExtract = Partition(RequestedGVs); + + // Take a 'None' partition to mean the whole module (as opposed to an empty + // partition, which means "materialize nothing"). Emit the whole module + // unmodified to the base layer. + if (GVsToExtract == None) { + Defs.clear(); + BaseLayer.emit(std::move(R), std::move(TSM)); + return; + } + + // If the partition is empty, return the whole module to the symbol table. + if (GVsToExtract->empty()) { + R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + std::move(TSM), R.getSymbols(), std::move(Defs), *this)); + return; + } + + // Ok -- we actually need to partition the symbols. Promote the symbol + // linkages/names. + // FIXME: We apply this once per partitioning. It's safe, but overkill. + { + auto PromotedGlobals = PromoteSymbols(*TSM.getModule()); + if (!PromotedGlobals.empty()) { + MangleAndInterner Mangle(ES, TSM.getModule()->getDataLayout()); + SymbolFlagsMap SymbolFlags; + for (auto &GV : PromotedGlobals) + SymbolFlags[Mangle(GV->getName())] = + JITSymbolFlags::fromGlobalValue(*GV); + if (auto Err = R.defineMaterializing(SymbolFlags)) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + } + } + + expandPartition(*GVsToExtract); + + // Extract the requested partiton (plus any necessary aliases) and + // put the rest back into the impl dylib. + auto ShouldExtract = [&](const GlobalValue &GV) -> bool { + return GVsToExtract->count(&GV); + }; + + auto ExtractedTSM = extractSubModule(TSM, ".submodule", ShouldExtract); + R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>( + ES, std::move(TSM), R.getVModuleKey(), *this)); + + BaseLayer.emit(std::move(R), std::move(ExtractedTSM)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp new file mode 100644 index 000000000000..d46b6fcf9a5f --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -0,0 +1,86 @@ +//===------ CompileUtils.cpp - Utilities for compiling IR in the JIT ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include <algorithm> + +namespace llvm { +namespace orc { + +/// Compile a Module to an ObjectFile. +SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { + CompileResult CachedObject = tryToLoadFromObjectCache(M); + if (CachedObject) + return CachedObject; + + SmallVector<char, 0> ObjBufferSV; + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + PM.run(M); + } + + auto ObjBuffer = llvm::make_unique<SmallVectorMemoryBuffer>( + std::move(ObjBufferSV), + "<in memory object compiled from " + M.getModuleIdentifier() + ">"); + + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (Obj) { + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); + } + + // TODO: Actually report errors helpfully. + consumeError(Obj.takeError()); + return nullptr; +} + +SimpleCompiler::CompileResult +SimpleCompiler::tryToLoadFromObjectCache(const Module &M) { + if (!ObjCache) + return CompileResult(); + + return ObjCache->getObject(&M); +} + +void SimpleCompiler::notifyObjectCompiled(const Module &M, + const MemoryBuffer &ObjBuffer) { + if (ObjCache) + ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); +} + +ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache) + : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + +std::unique_ptr<MemoryBuffer> ConcurrentIRCompiler::operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp new file mode 100644 index 000000000000..dac37e030e0c --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -0,0 +1,1920 @@ +//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" + +#if LLVM_ENABLE_THREADS +#include <future> +#endif + +#define DEBUG_TYPE "orc" + +using namespace llvm; + +namespace { + +#ifndef NDEBUG + +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), + cl::desc("debug print hidden symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), + cl::desc("debug print callable symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), + cl::desc("debug print data symbols defined by " + "materialization units"), + cl::Hidden); + +#endif // NDEBUG + +// SetPrinter predicate that prints every element. +template <typename T> struct PrintAll { + bool operator()(const T &E) { return true; } +}; + +bool anyPrintSymbolOptionSet() { +#ifndef NDEBUG + return PrintHidden || PrintCallable || PrintData; +#else + return false; +#endif // NDEBUG +} + +bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { +#ifndef NDEBUG + // Bail out early if this is a hidden symbol and we're not printing hiddens. + if (!PrintHidden && !Flags.isExported()) + return false; + + // Return true if this is callable and we're printing callables. + if (PrintCallable && Flags.isCallable()) + return true; + + // Return true if this is data and we're printing data. + if (PrintData && !Flags.isCallable()) + return true; + + // otherwise return false. + return false; +#else + return false; +#endif // NDEBUG +} + +// Prints a set of items, filtered by an user-supplied predicate. +template <typename Set, typename Pred = PrintAll<typename Set::value_type>> +class SetPrinter { +public: + SetPrinter(const Set &S, Pred ShouldPrint = Pred()) + : S(S), ShouldPrint(std::move(ShouldPrint)) {} + + void printTo(llvm::raw_ostream &OS) const { + bool PrintComma = false; + OS << "{"; + for (auto &E : S) { + if (ShouldPrint(E)) { + if (PrintComma) + OS << ','; + OS << ' ' << E; + PrintComma = true; + } + } + OS << " }"; + } + +private: + const Set &S; + mutable Pred ShouldPrint; +}; + +template <typename Set, typename Pred> +SetPrinter<Set, Pred> printSet(const Set &S, Pred P = Pred()) { + return SetPrinter<Set, Pred>(S, std::move(P)); +} + +// Render a SetPrinter by delegating to its printTo method. +template <typename Set, typename Pred> +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, + const SetPrinter<Set, Pred> &Printer) { + Printer.printTo(OS); + return OS; +} + +struct PrintSymbolFlagsMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolFlagsMap::value_type &KV) { + return flagsMatchCLOpts(KV.second); + } +}; + +struct PrintSymbolMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolMap::value_type &KV) { + return flagsMatchCLOpts(KV.second.getFlags()); + } +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +char FailedToMaterialize::ID = 0; +char SymbolsNotFound::ID = 0; +char SymbolsCouldNotBeRemoved::ID = 0; + +RegisterDependenciesFunction NoDependenciesToRegister = + RegisterDependenciesFunction(); + +void MaterializationUnit::anchor() {} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { + return OS << *Sym; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + return OS << printSet(Symbols, PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.isCallable()) + OS << "[Callable]"; + else + OS << "[Data]"; + if (Flags.isWeak()) + OS << "[Weak]"; + else if (Flags.isCommon()) + OS << "[Common]"; + + if (!Flags.isExported()) + OS << "[Hidden]"; + + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { + return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " + << Sym.getFlags(); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { + return OS << "(\"" << KV.first << "\", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { + return OS << "(\"" << KV.first << "\": " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + return OS << printSet(SymbolFlags, PrintSymbolFlagsMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { + return OS << printSet(Symbols, PrintSymbolMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV) { + return OS << "(" << KV.first << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { + return OS << printSet(Deps, PrintAll<SymbolDependenceMap::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { + OS << "MU@" << &MU << " (\"" << MU.getName() << "\""; + if (anyPrintSymbolOptionSet()) + OS << ", " << MU.getSymbols(); + return OS << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { + OS << "["; + if (!JDs.empty()) { + assert(JDs.front().first && "JITDylibList entries must not be null"); + OS << " (\"" << JDs.front().first->getName() << "\", " + << (JDs.front().second ? "true" : "false") << ")"; + for (auto &KV : make_range(std::next(JDs.begin()), JDs.end())) { + assert(KV.first && "JITDylibList entries must not be null"); + OS << ", (\"" << KV.first->getName() << "\", " + << (KV.second ? "true" : "false") << ")"; + } + } + OS << " ]"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }\n"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + +FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code FailedToMaterialize::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void FailedToMaterialize::log(raw_ostream &OS) const { + OS << "Failed to materialize symbols: " << Symbols; +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsNotFound::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsNotFound::log(raw_ostream &OS) const { + OS << "Symbols not found: " << Symbols; +} + +SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { + OS << "Symbols could not be removed: " << Symbols; +} + +AsynchronousSymbolQuery::AsynchronousSymbolQuery( + const SymbolNameSet &Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete) + : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { + assert(RequiredState >= SymbolState::Resolved && + "Cannot query for a symbols that have not reached the resolve state " + "yet"); + + OutstandingSymbolsCount = Symbols.size(); + + for (auto &S : Symbols) + ResolvedSymbols[S] = nullptr; +} + +void AsynchronousSymbolQuery::notifySymbolMetRequiredState( + const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Resolving symbol outside the requested set"); + assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + I->second = std::move(Sym); + --OutstandingSymbolsCount; +} + +void AsynchronousSymbolQuery::handleComplete() { + assert(OutstandingSymbolsCount == 0 && + "Symbols remain, handleComplete called prematurely"); + + auto TmpNotifyComplete = std::move(NotifyComplete); + NotifyComplete = SymbolsResolvedCallback(); + TmpNotifyComplete(std::move(ResolvedSymbols)); +} + +bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } + +void AsynchronousSymbolQuery::handleFailed(Error Err) { + assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && + OutstandingSymbolsCount == 0 && + "Query should already have been abandoned"); + NotifyComplete(std::move(Err)); + NotifyComplete = SymbolsResolvedCallback(); +} + +void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, + SymbolStringPtr Name) { + bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second; + (void)Added; + assert(Added && "Duplicate dependence notification?"); +} + +void AsynchronousSymbolQuery::removeQueryDependence( + JITDylib &JD, const SymbolStringPtr &Name) { + auto QRI = QueryRegistrations.find(&JD); + assert(QRI != QueryRegistrations.end() && + "No dependencies registered for JD"); + assert(QRI->second.count(Name) && "No dependency on Name in JD"); + QRI->second.erase(Name); + if (QRI->second.empty()) + QueryRegistrations.erase(QRI); +} + +void AsynchronousSymbolQuery::detach() { + ResolvedSymbols.clear(); + OutstandingSymbolsCount = 0; + for (auto &KV : QueryRegistrations) + KV.first->detachQueryHelper(*this, KV.second); + QueryRegistrations.clear(); +} + +MaterializationResponsibility::MaterializationResponsibility( + JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) + : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); +} + +MaterializationResponsibility::~MaterializationResponsibility() { + assert(SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); +} + +SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { + return JD.getRequestedSymbols(SymbolFlags); +} + +void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; + }); +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto I = SymbolFlags.find(KV.first); + assert(I != SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + if (I->second.isWeak()) + assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && + "Resolving symbol with incorrect flags"); + else + assert(I->second == KV.second.getFlags() && + "Resolving symbol with incorrect flags"); + } +#endif + + JD.resolve(Symbols); +} + +void MaterializationResponsibility::notifyEmitted() { + + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; + }); + + JD.emit(SymbolFlags); + SymbolFlags.clear(); +} + +Error MaterializationResponsibility::defineMaterializing( + const SymbolFlagsMap &NewSymbolFlags) { + // Add the given symbols to this responsibility object. + // It's ok if we hit a duplicate here: In that case the new version will be + // discarded, and the JITDylib::defineMaterializing method will return a + // duplicate symbol error. + for (auto &KV : NewSymbolFlags) + SymbolFlags.insert(KV); + + return JD.defineMaterializing(NewSymbolFlags); +} + +void MaterializationResponsibility::failMaterialization() { + + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " failing materialization for " + << SymbolFlags << "\n"; + }); + + SymbolNameSet FailedSymbols; + for (auto &KV : SymbolFlags) + FailedSymbols.insert(KV.first); + + JD.notifyFailed(FailedSymbols); + SymbolFlags.clear(); +} + +void MaterializationResponsibility::replace( + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) + SymbolFlags.erase(KV.first); + + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD.getName() << " replacing symbols with " << *MU + << "\n"; + });); + + JD.replace(std::move(MU)); +} + +MaterializationResponsibility +MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, + VModuleKey NewKey) { + + if (NewKey == VModuleKey()) + NewKey = K; + + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = SymbolFlags.find(Name); + assert(I != SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + SymbolFlags.erase(I); + } + + return MaterializationResponsibility(JD, std::move(DelegatedFlags), + std::move(NewKey)); +} + +void MaterializationResponsibility::addDependencies( + const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { + assert(SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + JD.addDependencies(Name, Dependencies); +} + +void MaterializationResponsibility::addDependenciesForAll( + const SymbolDependenceMap &Dependencies) { + for (auto &KV : SymbolFlags) + JD.addDependencies(KV.first, Dependencies); +} + +AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( + SymbolMap Symbols, VModuleKey K) + : MaterializationUnit(extractFlags(Symbols), std::move(K)), + Symbols(std::move(Symbols)) {} + +StringRef AbsoluteSymbolsMaterializationUnit::getName() const { + return "<Absolute Symbols>"; +} + +void AbsoluteSymbolsMaterializationUnit::materialize( + MaterializationResponsibility R) { + R.notifyResolved(Symbols); + R.notifyEmitted(); +} + +void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(Symbols.count(Name) && "Symbol is not part of this MU"); + Symbols.erase(Name); +} + +SymbolFlagsMap +AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { + SymbolFlagsMap Flags; + for (const auto &KV : Symbols) + Flags[KV.first] = KV.second.getFlags(); + return Flags; +} + +ReExportsMaterializationUnit::ReExportsMaterializationUnit( + JITDylib *SourceJD, bool MatchNonExported, SymbolAliasMap Aliases, + VModuleKey K) + : MaterializationUnit(extractFlags(Aliases), std::move(K)), + SourceJD(SourceJD), MatchNonExported(MatchNonExported), + Aliases(std::move(Aliases)) {} + +StringRef ReExportsMaterializationUnit::getName() const { + return "<Reexports>"; +} + +void ReExportsMaterializationUnit::materialize( + MaterializationResponsibility R) { + + auto &ES = R.getTargetJITDylib().getExecutionSession(); + JITDylib &TgtJD = R.getTargetJITDylib(); + JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD; + + // Find the set of requested aliases and aliasees. Return any unrequested + // aliases back to the JITDylib so as to not prematurely materialize any + // aliasees. + auto RequestedSymbols = R.getRequestedSymbols(); + SymbolAliasMap RequestedAliases; + + for (auto &Name : RequestedSymbols) { + auto I = Aliases.find(Name); + assert(I != Aliases.end() && "Symbol not found in aliases map?"); + RequestedAliases[Name] = std::move(I->second); + Aliases.erase(I); + } + + LLVM_DEBUG({ + ES.runSessionLocked([&]() { + dbgs() << "materializing reexports: target = " << TgtJD.getName() + << ", source = " << SrcJD.getName() << " " << RequestedAliases + << "\n"; + }); + }); + + if (!Aliases.empty()) { + if (SourceJD) + R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); + else + R.replace(symbolAliases(std::move(Aliases))); + } + + // The OnResolveInfo struct will hold the aliases and responsibilty for each + // query in the list. + struct OnResolveInfo { + OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases) + : R(std::move(R)), Aliases(std::move(Aliases)) {} + + MaterializationResponsibility R; + SymbolAliasMap Aliases; + }; + + // Build a list of queries to issue. In each round we build the largest set of + // aliases that we can resolve without encountering a chain definition of the + // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would + // be waitin on a symbol that it itself had to resolve. Usually this will just + // involve one round and a single query. + + std::vector<std::pair<SymbolNameSet, std::shared_ptr<OnResolveInfo>>> + QueryInfos; + while (!RequestedAliases.empty()) { + SymbolNameSet ResponsibilitySymbols; + SymbolNameSet QuerySymbols; + SymbolAliasMap QueryAliases; + + // Collect as many aliases as we can without including a chain. + for (auto &KV : RequestedAliases) { + // Chain detected. Skip this symbol for this round. + if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) || + RequestedAliases.count(KV.second.Aliasee))) + continue; + + ResponsibilitySymbols.insert(KV.first); + QuerySymbols.insert(KV.second.Aliasee); + QueryAliases[KV.first] = std::move(KV.second); + } + + // Remove the aliases collected this round from the RequestedAliases map. + for (auto &KV : QueryAliases) + RequestedAliases.erase(KV.first); + + assert(!QuerySymbols.empty() && "Alias cycle detected!"); + + auto QueryInfo = std::make_shared<OnResolveInfo>( + R.delegate(ResponsibilitySymbols), std::move(QueryAliases)); + QueryInfos.push_back( + make_pair(std::move(QuerySymbols), std::move(QueryInfo))); + } + + // Issue the queries. + while (!QueryInfos.empty()) { + auto QuerySymbols = std::move(QueryInfos.back().first); + auto QueryInfo = std::move(QueryInfos.back().second); + + QueryInfos.pop_back(); + + auto RegisterDependencies = [QueryInfo, + &SrcJD](const SymbolDependenceMap &Deps) { + // If there were no materializing symbols, just bail out. + if (Deps.empty()) + return; + + // Otherwise the only deps should be on SrcJD. + assert(Deps.size() == 1 && Deps.count(&SrcJD) && + "Unexpected dependencies for reexports"); + + auto &SrcJDDeps = Deps.find(&SrcJD)->second; + SymbolDependenceMap PerAliasDepsMap; + auto &PerAliasDeps = PerAliasDepsMap[&SrcJD]; + + for (auto &KV : QueryInfo->Aliases) + if (SrcJDDeps.count(KV.second.Aliasee)) { + PerAliasDeps = {KV.second.Aliasee}; + QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); + } + }; + + auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { + if (Result) { + SymbolMap ResolutionMap; + for (auto &KV : QueryInfo->Aliases) { + assert(Result->count(KV.second.Aliasee) && + "Result map missing entry?"); + ResolutionMap[KV.first] = JITEvaluatedSymbol( + (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + } + QueryInfo->R.notifyResolved(ResolutionMap); + QueryInfo->R.notifyEmitted(); + } else { + auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); + ES.reportError(Result.takeError()); + QueryInfo->R.failMaterialization(); + } + }; + + ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, + SymbolState::Resolved, std::move(OnComplete), + std::move(RegisterDependencies)); + } +} + +void ReExportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(Aliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + Aliases.erase(Name); +} + +SymbolFlagsMap +ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) + SymbolFlags[KV.first] = KV.second.AliasFlags; + + return SymbolFlags; +} + +Expected<SymbolAliasMap> +buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { + auto Flags = SourceJD.lookupFlags(Symbols); + + if (!Flags) + return Flags.takeError(); + + if (Flags->size() != Symbols.size()) { + SymbolNameSet Unresolved = Symbols; + for (auto &KV : *Flags) + Unresolved.erase(KV.first); + return make_error<SymbolsNotFound>(std::move(Unresolved)); + } + + SymbolAliasMap Result; + for (auto &Name : Symbols) { + assert(Flags->count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]); + } + + return Result; +} + +ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, + bool MatchNonExported, + SymbolPredicate Allow) + : SourceJD(SourceJD), MatchNonExported(MatchNonExported), + Allow(std::move(Allow)) {} + +Expected<SymbolNameSet> +ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { + orc::SymbolNameSet Added; + orc::SymbolAliasMap AliasMap; + + auto Flags = SourceJD.lookupFlags(Names); + + if (!Flags) + return Flags.takeError(); + + for (auto &KV : *Flags) { + if (Allow && !Allow(KV.first)) + continue; + AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); + Added.insert(KV.first); + } + + if (!Added.empty()) + cantFail(JD.define(reexports(SourceJD, AliasMap, MatchNonExported))); + + return Added; +} + +Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { + return ES.runSessionLocked([&]() -> Error { + std::vector<SymbolTable::iterator> AddedSyms; + + for (auto &KV : SymbolFlags) { + SymbolTable::iterator EntryItr; + bool Added; + + std::tie(EntryItr, Added) = + Symbols.insert(std::make_pair(KV.first, SymbolTableEntry(KV.second))); + + if (Added) { + AddedSyms.push_back(EntryItr); + EntryItr->second.setState(SymbolState::Materializing); + } else { + // Remove any symbols already added. + for (auto &SI : AddedSyms) + Symbols.erase(SI); + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(*KV.first); + } + } + + return Error::success(); + }); +} + +void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { + assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + + auto MustRunMU = + ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> { + +#ifndef NDEBUG + for (auto &KV : MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI != Symbols.end() && "Replacing unknown symbol"); + assert(SymI->second.isInMaterializationPhase() && + "Can not call replace on a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Symbol should not have materializer attached already"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Symbol being replaced should have no UnmaterializedInfo"); + } +#endif // NDEBUG + + // If any symbol has pending queries against it then we need to + // materialize MU immediately. + for (auto &KV : MU->getSymbols()) { + auto MII = MaterializingInfos.find(KV.first); + if (MII != MaterializingInfos.end()) { + if (MII->second.hasQueriesPending()) + return std::move(MU); + } + } + + // Otherwise, make MU responsible for all the symbols. + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + for (auto &KV : UMI->MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Can not replace a symbol that has a materializer attached"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Unexpected materializer entry in map"); + SymI->second.setAddress(SymI->second.getAddress()); + SymI->second.setMaterializerAttached(true); + UnmaterializedInfos[KV.first] = UMI; + } + + return nullptr; + }); + + if (MustRunMU) + ES.dispatchMaterialization(*this, std::move(MustRunMU)); +} + +SymbolNameSet +JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { + return ES.runSessionLocked([&]() { + SymbolNameSet RequestedSymbols; + + for (auto &KV : SymbolFlags) { + assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); + assert(Symbols.find(KV.first)->second.isInMaterializationPhase() && + "getRequestedSymbols can only be called for symbols that have " + "started materializing"); + auto I = MaterializingInfos.find(KV.first); + if (I == MaterializingInfos.end()) + continue; + + if (I->second.hasQueriesPending()) + RequestedSymbols.insert(KV.first); + } + + return RequestedSymbols; + }); +} + +void JITDylib::addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + assert(Symbols.count(Name) && "Name not in symbol table"); + assert(Symbols[Name].isInMaterializationPhase() && + "Can not add dependencies for a symbol that is not materializing"); + + auto &MI = MaterializingInfos[Name]; + assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); + + for (auto &KV : Dependencies) { + assert(KV.first && "Null JITDylib in dependency?"); + auto &OtherJITDylib = *KV.first; + auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; + + for (auto &OtherSymbol : KV.second) { +#ifndef NDEBUG + // Assert that this symbol exists and has not been emitted already. + auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); + assert(SymI != OtherJITDylib.Symbols.end() && + (SymI->second.getState() != SymbolState::Ready && + "Dependency on emitted symbol")); +#endif + + auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; + + if (OtherMI.IsEmitted) + transferEmittedNodeDependencies(MI, Name, OtherMI); + else if (&OtherJITDylib != this || OtherSymbol != Name) { + OtherMI.Dependants[this].insert(Name); + DepsOnOtherJITDylib.insert(OtherSymbol); + } + } + + if (DepsOnOtherJITDylib.empty()) + MI.UnemittedDependencies.erase(&OtherJITDylib); + } +} + +void JITDylib::resolve(const SymbolMap &Resolved) { + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; + for (const auto &KV : Resolved) { + auto &Name = KV.first; + auto Sym = KV.second; + + auto I = Symbols.find(Name); + + assert(I != Symbols.end() && "Symbol not found"); + assert(!I->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(I->second.getState() == SymbolState::Materializing && + "Symbol should be materializing"); + assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); + + assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == + (I->second.getFlags() & ~JITSymbolFlags::Weak) && + "Resolved flags should match the declared flags"); + + // Once resolved, symbols can never be weak. + JITSymbolFlags ResolvedFlags = Sym.getFlags(); + ResolvedFlags &= ~JITSymbolFlags::Weak; + I->second.setAddress(Sym.getAddress()); + I->second.setFlags(ResolvedFlags); + I->second.setState(SymbolState::Resolved); + + auto &MI = MaterializingInfos[Name]; + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, Sym); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); + } + } + + return CompletedQueries; + }); + + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q not completed"); + Q->handleComplete(); + } +} + +void JITDylib::emit(const SymbolFlagsMap &Emitted) { + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; + + for (const auto &KV : Emitted) { + const auto &Name = KV.first; + + auto MII = MaterializingInfos.find(Name); + assert(MII != MaterializingInfos.end() && + "Missing MaterializingInfo entry"); + + auto &MI = MII->second; + + // For each dependant, transfer this node's emitted dependencies to + // it. If the dependant node is ready (i.e. has no unemitted + // dependencies) then notify any pending queries. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantJD.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantJD.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); + + auto &DependantMI = DependantMII->second; + + // Remove the dependant's dependency on this node. + assert(DependantMI.UnemittedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); + DependantMI.UnemittedDependencies[this].erase(Name); + if (DependantMI.UnemittedDependencies[this].empty()) + DependantMI.UnemittedDependencies.erase(this); + + // Transfer unemitted dependencies from this node to the dependant. + DependantJD.transferEmittedNodeDependencies(DependantMI, + DependantName, MI); + + // If the dependant is emitted and this node was the last of its + // unemitted dependencies then the dependant node is now ready, so + // notify any pending queries on the dependant node. + if (DependantMI.IsEmitted && + DependantMI.UnemittedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); + + // Since this dependant is now ready, we erase its MaterializingInfo + // and update its materializing state. + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && + "Dependant has no entry in the Symbols table"); + DependantSymI->second.setState(SymbolState::Ready); + + for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + + DependantJD.MaterializingInfos.erase(DependantMII); + } + } + } + MI.Dependants.clear(); + MI.IsEmitted = true; + + if (MI.UnemittedDependencies.empty()) { + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table"); + SymI->second.setState(SymbolState::Ready); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(*this, Name); + } + MaterializingInfos.erase(MII); + } + } + + return CompletedQueries; + }); + + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q is not complete"); + Q->handleComplete(); + } +} + +void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { + + // FIXME: This should fail any transitively dependant symbols too. + + auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet FailedQueries; + std::vector<MaterializingInfosMap::iterator> MIIsToRemove; + + for (auto &Name : FailedSymbols) { + auto I = Symbols.find(Name); + assert(I != Symbols.end() && "Symbol not present in this JITDylib"); + Symbols.erase(I); + + auto MII = MaterializingInfos.find(Name); + + // If we have not created a MaterializingInfo for this symbol yet then + // there is nobody to notify. + if (MII == MaterializingInfos.end()) + continue; + + // Remove this symbol from the dependants list of any dependencies. + for (auto &KV : MII->second.UnemittedDependencies) { + auto *DependencyJD = KV.first; + auto &Dependencies = KV.second; + for (auto &DependencyName : Dependencies) { + auto DependencyMII = + DependencyJD->MaterializingInfos.find(DependencyName); + assert(DependencyMII != DependencyJD->MaterializingInfos.end() && + "Unemitted dependency must have a MaterializingInfo entry"); + assert(DependencyMII->second.Dependants.count(this) && + "Dependency's dependants list does not contain this JITDylib"); + assert(DependencyMII->second.Dependants[this].count(Name) && + "Dependency's dependants list does not contain dependant"); + DependencyMII->second.Dependants[this].erase(Name); + } + } + + // Copy all the queries to the FailedQueries list, then abandon them. + // This has to be a copy, and the copy has to come before the abandon + // operation: Each Q.detach() call will reach back into this + // PendingQueries list to remove Q. + for (auto &Q : MII->second.pendingQueries()) + FailedQueries.insert(Q); + + MIIsToRemove.push_back(std::move(MII)); + } + + // Detach failed queries. + for (auto &Q : FailedQueries) + Q->detach(); + + // Remove the MaterializingInfos. + for (auto &MII : MIIsToRemove) { + assert(!MII->second.hasQueriesPending() && + "Queries remain after symbol was failed"); + + MaterializingInfos.erase(MII); + } + + return FailedQueries; + }); + + for (auto &Q : FailedQueriesToNotify) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, + bool SearchThisJITDylibFirst, + bool MatchNonExportedInThisDylib) { + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } + + ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); +} + +void JITDylib::addToSearchOrder(JITDylib &JD, bool MatchNonExported) { + ES.runSessionLocked([&]() { + SearchOrder.push_back({&JD, MatchNonExported}); + }); +} + +void JITDylib::replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, + bool MatchNonExported) { + ES.runSessionLocked([&]() { + auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), + [&](const JITDylibSearchList::value_type &KV) { + return KV.first == &OldJD; + }); + + if (I != SearchOrder.end()) + *I = {&NewJD, MatchNonExported}; + }); +} + +void JITDylib::removeFromSearchOrder(JITDylib &JD) { + ES.runSessionLocked([&]() { + auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), + [&](const JITDylibSearchList::value_type &KV) { + return KV.first == &JD; + }); + if (I != SearchOrder.end()) + SearchOrder.erase(I); + }); +} + +Error JITDylib::remove(const SymbolNameSet &Names) { + return ES.runSessionLocked([&]() -> Error { + using SymbolMaterializerItrPair = + std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>; + std::vector<SymbolMaterializerItrPair> SymbolsToRemove; + SymbolNameSet Missing; + SymbolNameSet Materializing; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + + // Note symbol missing. + if (I == Symbols.end()) { + Missing.insert(Name); + continue; + } + + // Note symbol materializing. + if (I->second.isInMaterializationPhase()) { + Materializing.insert(Name); + continue; + } + + auto UMII = I->second.hasMaterializerAttached() + ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); + SymbolsToRemove.push_back(std::make_pair(I, UMII)); + } + + // If any of the symbols are not defined, return an error. + if (!Missing.empty()) + return make_error<SymbolsNotFound>(std::move(Missing)); + + // If any of the symbols are currently materializing, return an error. + if (!Materializing.empty()) + return make_error<SymbolsCouldNotBeRemoved>(std::move(Materializing)); + + // Remove the symbols. + for (auto &SymbolMaterializerItrPair : SymbolsToRemove) { + auto UMII = SymbolMaterializerItrPair.second; + + // If there is a materializer attached, call discard. + if (UMII != UnmaterializedInfos.end()) { + UMII->second->MU->doDiscard(*this, UMII->first); + UnmaterializedInfos.erase(UMII); + } + + auto SymI = SymbolMaterializerItrPair.first; + Symbols.erase(SymI); + } + + return Error::success(); + }); +} + +Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { + SymbolFlagsMap Result; + auto Unresolved = lookupFlagsImpl(Result, Names); + if (!Unresolved) + return Unresolved.takeError(); + + if (DefGenerator && !Unresolved->empty()) { + auto NewDefs = DefGenerator(*this, *Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); + if (!Unresolved2) + return Unresolved2.takeError(); + (void)Unresolved2; + assert(Unresolved2->empty() && + "All fallback defs should have been found by lookupFlagsImpl"); + } + }; + return Result; + }); +} + +Expected<SymbolNameSet> JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { + SymbolNameSet Unresolved; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + if (I != Symbols.end()) { + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = I->second.getFlags(); + } else + Unresolved.insert(Name); + } + + return Unresolved; +} + +Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs) { + assert(Q && "Query can not be null"); + + lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); + if (DefGenerator && !Unresolved.empty()) { + auto NewDefs = DefGenerator(*this, Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) + Unresolved.erase(D); + lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs); + assert(NewDefs->empty() && + "All fallback defs should have been found by lookupImpl"); + } + } + + return Error::success(); +} + +void JITDylib::lodgeQueryImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, + bool MatchNonExported, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { + + std::vector<SymbolStringPtr> ToRemove; + for (auto Name : Unresolved) { + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If this is a non exported symbol and we're skipping those then skip it. + if (!SymI->second.getFlags().isExported() && !MatchNonExported) + continue; + + // If we matched against Name in JD, mark it to be removed from the + // Unresolved set. + ToRemove.push_back(Name); + + // If this symbol already meets the required state for then notify the + // query and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + continue; + } + + // Otherwise this symbol does not yet meet the required state. Check whether + // it has a materializer attached, and if so prepare to run it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Symbol not resolved but already has address?"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } + + // Add the query to the PendingQueries list. + assert(SymI->second.isInMaterializationPhase() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(*this, Name); + } + + // Remove any symbols that we found. + for (auto &Name : ToRemove) + Unresolved.erase(Name); +} + +Expected<SymbolNameSet> +JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { + assert(Q && "Query can not be null"); + + ES.runOutstandingMUs(); + + bool QueryComplete = false; + std::vector<std::unique_ptr<MaterializationUnit>> MUs; + + SymbolNameSet Unresolved = std::move(Names); + auto Err = ES.runSessionLocked([&, this]() -> Error { + QueryComplete = lookupImpl(Q, MUs, Unresolved); + if (DefGenerator && !Unresolved.empty()) { + assert(!QueryComplete && "query complete but unresolved symbols remain?"); + auto NewDefs = DefGenerator(*this, Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) + Unresolved.erase(D); + QueryComplete = lookupImpl(Q, MUs, *NewDefs); + assert(NewDefs->empty() && + "All fallback defs should have been found by lookupImpl"); + } + } + return Error::success(); + }); + + if (Err) + return std::move(Err); + + assert((MUs.empty() || !QueryComplete) && + "If action flags are set, there should be no work to do (so no MUs)"); + + if (QueryComplete) + Q->handleComplete(); + + // FIXME: Swap back to the old code below once RuntimeDyld works with + // callbacks from asynchronous queries. + // Add MUs to the OutstandingMUs list. + { + std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); + for (auto &MU : MUs) + ES.OutstandingMUs.push_back(make_pair(this, std::move(MU))); + } + ES.runOutstandingMUs(); + + // Dispatch any required MaterializationUnits for materialization. + // for (auto &MU : MUs) + // ES.dispatchMaterialization(*this, std::move(MU)); + + return Unresolved; +} + +bool JITDylib::lookupImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { + bool QueryComplete = false; + + std::vector<SymbolStringPtr> ToRemove; + for (auto Name : Unresolved) { + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If we found Name, mark it to be removed from the Unresolved set. + ToRemove.push_back(Name); + + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + QueryComplete = true; + continue; + } + + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Kick all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + assert(SymK != Symbols.end() && "Missing symbol table entry"); + SymK->second.setState(SymbolState::Materializing); + SymK->second.setMaterializerAttached(false); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } + + // Add the query to the PendingQueries list. + assert(SymI->second.isInMaterializationPhase() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(*this, Name); + } + + // Remove any marked symbols from the Unresolved set. + for (auto &Name : ToRemove) + Unresolved.erase(Name); + + return QueryComplete; +} + +void JITDylib::dump(raw_ostream &OS) { + ES.runSessionLocked([&, this]() { + OS << "JITDylib \"" << JITDylibName << "\" (ES: " + << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n" + << "Search order: ["; + for (auto &KV : SearchOrder) + OS << " (\"" << KV.first->getName() << "\", " + << (KV.second ? "all" : "exported only") << ")"; + OS << " ]\n" + << "Symbol table:\n"; + + for (auto &KV : Symbols) { + OS << " \"" << *KV.first << "\": "; + if (auto Addr = KV.second.getAddress()) + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags() + << " "; + else + OS << "<not resolved> "; + + OS << KV.second.getState(); + + if (KV.second.hasMaterializerAttached()) { + OS << " (Materializer "; + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << I->second->MU.get() << ")\n"; + } else + OS << "\n"; + } + + if (!MaterializingInfos.empty()) + OS << " MaterializingInfos entries:\n"; + for (auto &KV : MaterializingInfos) { + OS << " \"" << *KV.first << "\":\n" + << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") + << "\n" + << " " << KV.second.pendingQueries().size() + << " pending queries: { "; + for (const auto &Q : KV.second.pendingQueries()) + OS << Q.get() << " (" << Q->getRequiredState() << ") "; + OS << "}\n Dependants:\n"; + for (auto &KV2 : KV.second.Dependants) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + OS << " Unemitted Dependencies:\n"; + for (auto &KV2 : KV.second.UnemittedDependencies) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + } + }); +} + +void JITDylib::MaterializingInfo::addQuery( + std::shared_ptr<AsynchronousSymbolQuery> Q) { + + auto I = std::lower_bound( + PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) { + return V->getRequiredState() <= S; + }); + PendingQueries.insert(I.base(), std::move(Q)); +} + +void JITDylib::MaterializingInfo::removeQuery( + const AsynchronousSymbolQuery &Q) { + // FIXME: Implement 'find_as' for shared_ptr<T>/T*. + auto I = + std::find_if(PendingQueries.begin(), PendingQueries.end(), + [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); + assert(I != PendingQueries.end() && + "Query is not attached to this MaterializingInfo"); + PendingQueries.erase(I); +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { + AsynchronousSymbolQueryList Result; + while (!PendingQueries.empty()) { + if (PendingQueries.back()->getRequiredState() > RequiredState) + break; + + Result.push_back(std::move(PendingQueries.back())); + PendingQueries.pop_back(); + } + + return Result; +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeAllQueries() { + AsynchronousSymbolQueryList Result; + std::swap(Result, PendingQueries); + return Result; +} + +JITDylib::JITDylib(ExecutionSession &ES, std::string Name) + : ES(ES), JITDylibName(std::move(Name)) { + SearchOrder.push_back({this, true}); +} + +Error JITDylib::defineImpl(MaterializationUnit &MU) { + SymbolNameSet Duplicates; + std::vector<SymbolStringPtr> ExistingDefsOverridden; + std::vector<SymbolStringPtr> MUDefsOverridden; + + for (const auto &KV : MU.getSymbols()) { + auto I = Symbols.find(KV.first); + + if (I != Symbols.end()) { + if (KV.second.isStrong()) { + if (I->second.getFlags().isStrong() || + I->second.getState() > SymbolState::NeverSearched) + Duplicates.insert(KV.first); + else { + assert(I->second.getState() == SymbolState::NeverSearched && + "Overridden existing def should be in the never-searched " + "state"); + ExistingDefsOverridden.push_back(KV.first); + } + } else + MUDefsOverridden.push_back(KV.first); + } + } + + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) + return make_error<DuplicateDefinition>(**Duplicates.begin()); + + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); + + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { + + auto UMII = UnmaterializedInfos.find(S); + assert(UMII != UnmaterializedInfos.end() && + "Overridden existing def should have an UnmaterializedInfo"); + UMII->second->MU->doDiscard(*this, S); + } + + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto &SymEntry = Symbols[KV.first]; + SymEntry.setFlags(KV.second); + SymEntry.setState(SymbolState::NeverSearched); + SymEntry.setMaterializerAttached(true); + } + + return Error::success(); +} + +void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols) { + for (auto &QuerySymbol : QuerySymbols) { + assert(MaterializingInfos.count(QuerySymbol) && + "QuerySymbol does not have MaterializingInfo"); + auto &MI = MaterializingInfos[QuerySymbol]; + MI.removeQuery(Q); + } +} + +void JITDylib::transferEmittedNodeDependencies( + MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName, + MaterializingInfo &EmittedMI) { + for (auto &KV : EmittedMI.UnemittedDependencies) { + auto &DependencyJD = *KV.first; + SymbolNameSet *UnemittedDependenciesOnDependencyJD = nullptr; + + for (auto &DependencyName : KV.second) { + auto &DependencyMI = DependencyJD.MaterializingInfos[DependencyName]; + + // Do not add self dependencies. + if (&DependencyMI == &DependantMI) + continue; + + // If we haven't looked up the dependencies for DependencyJD yet, do it + // now and cache the result. + if (!UnemittedDependenciesOnDependencyJD) + UnemittedDependenciesOnDependencyJD = + &DependantMI.UnemittedDependencies[&DependencyJD]; + + DependencyMI.Dependants[this].insert(DependantName); + UnemittedDependenciesOnDependencyJD->insert(DependencyName); + } + } +} + +ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { + // Construct the main dylib. + JDs.push_back(std::unique_ptr<JITDylib>(new JITDylib(*this, "<main>"))); +} + +JITDylib &ExecutionSession::getMainJITDylib() { + return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); +} + +JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { + return runSessionLocked([&, this]() -> JITDylib * { + for (auto &JD : JDs) + if (JD->getName() == Name) + return JD.get(); + return nullptr; + }); +} + +JITDylib &ExecutionSession::createJITDylib(std::string Name, + bool AddToMainDylibSearchOrder) { + assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); + return runSessionLocked([&, this]() -> JITDylib & { + JDs.push_back( + std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); + if (AddToMainDylibSearchOrder) + JDs.front()->addToSearchOrder(*JDs.back()); + return *JDs.back(); + }); +} + +void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { + assert(!!Err && "Error should be in failure state"); + + bool SendErrorToQuery; + runSessionLocked([&]() { + Q.detach(); + SendErrorToQuery = Q.canStillFail(); + }); + + if (SendErrorToQuery) + Q.handleFailed(std::move(Err)); + else + reportError(std::move(Err)); +} + +Expected<SymbolMap> ExecutionSession::legacyLookup( + LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + Error ResolutionError = Error::success(); + auto NotifyComplete = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); + PromisedResult.set_value(SymbolMap()); + } + }; +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + + auto NotifyComplete = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; +#endif + + auto Query = std::make_shared<AsynchronousSymbolQuery>( + Names, RequiredState, std::move(NotifyComplete)); + // FIXME: This should be run session locked along with the registration code + // and error reporting below. + SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); + + // If the query was lodged successfully then register the dependencies, + // otherwise fail it with an error. + if (UnresolvedSymbols.empty()) + RegisterDependencies(Query->QueryRegistrations); + else { + bool DeliverError = runSessionLocked([&]() { + Query->detach(); + return Query->canStillFail(); + }); + auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); + if (DeliverError) + Query->handleFailed(std::move(Err)); + else + reportError(std::move(Err)); + } + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + if (ResolutionError) + return std::move(ResolutionError); + return std::move(Result); + +#else + if (ResolutionError) + return std::move(ResolutionError); + + return Result; +#endif +} + +void ExecutionSession::lookup( + const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, + SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, + RegisterDependenciesFunction RegisterDependencies) { + + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Looking up " << Symbols << " in " << SearchOrder + << " (required state: " << RequiredState << ")\n"; + }); + }); + + // lookup can be re-entered recursively if running on a single thread. Run any + // outstanding MUs in case this query depends on them, otherwise this lookup + // will starve waiting for a result from an MU that is stuck in the queue. + runOutstandingMUs(); + + auto Unresolved = std::move(Symbols); + std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; + auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, + std::move(NotifyComplete)); + bool QueryComplete = false; + + auto LodgingErr = runSessionLocked([&]() -> Error { + auto LodgeQuery = [&]() -> Error { + for (auto &KV : SearchOrder) { + assert(KV.first && "JITDylibList entries must not be null"); + assert(!CollectedMUsMap.count(KV.first) && + "JITDylibList should not contain duplicate entries"); + + auto &JD = *KV.first; + auto MatchNonExported = KV.second; + if (auto Err = JD.lodgeQuery(Q, Unresolved, MatchNonExported, + CollectedMUsMap[&JD])) + return Err; + } + + if (!Unresolved.empty()) + return make_error<SymbolsNotFound>(std::move(Unresolved)); + + return Error::success(); + }; + + if (auto Err = LodgeQuery()) { + // Query failed. + + // Disconnect the query from its dependencies. + Q->detach(); + + // Replace the MUs. + for (auto &KV : CollectedMUsMap) + for (auto &MU : KV.second) + KV.first->replace(std::move(MU)); + + return Err; + } + + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryComplete = Q->isComplete(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + + return Error::success(); + }); + + if (LodgingErr) { + Q->handleFailed(std::move(LodgingErr)); + return; + } + + if (QueryComplete) + Q->handleComplete(); + + // Move the MUs to the OutstandingMUs list, then materialize. + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + for (auto &KV : CollectedMUsMap) + for (auto &MU : KV.second) + OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); + } + + runOutstandingMUs(); +} + +Expected<SymbolMap> +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + const SymbolNameSet &Symbols, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + Error ResolutionError = Error::success(); + + auto NotifyComplete = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); + PromisedResult.set_value(SymbolMap()); + } + }; + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + + auto NotifyComplete = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; +#endif + + // Perform the asynchronous lookup. + lookup(SearchOrder, Symbols, RequiredState, NotifyComplete, + RegisterDependencies); + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + if (ResolutionError) + return std::move(ResolutionError); + + return std::move(Result); + +#else + if (ResolutionError) + return std::move(ResolutionError); + + return Result; +#endif +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + SymbolStringPtr Name) { + SymbolNameSet Names({Name}); + + if (auto ResultMap = lookup(SearchOrder, std::move(Names), SymbolState::Ready, + NoDependenciesToRegister)) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, + SymbolStringPtr Name) { + SymbolNameSet Names({Name}); + + JITDylibSearchList FullSearchOrder; + FullSearchOrder.reserve(SearchOrder.size()); + for (auto *JD : SearchOrder) + FullSearchOrder.push_back({JD, false}); + + return lookup(FullSearchOrder, Name); +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name) { + return lookup(SearchOrder, intern(Name)); +} + +void ExecutionSession::dump(raw_ostream &OS) { + runSessionLocked([this, &OS]() { + for (auto &JD : JDs) + JD->dump(OS); + }); +} + +void ExecutionSession::runOutstandingMUs() { + while (1) { + std::pair<JITDylib *, std::unique_ptr<MaterializationUnit>> JITDylibAndMU; + + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + if (!OutstandingMUs.empty()) { + JITDylibAndMU = std::move(OutstandingMUs.back()); + OutstandingMUs.pop_back(); + } + } + + if (JITDylibAndMU.first) { + assert(JITDylibAndMU.second && "JITDylib, but no MU?"); + dispatchMaterialization(*JITDylibAndMU.first, + std::move(JITDylibAndMU.second)); + } else + break; + } +} + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.intern(MangledName); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp new file mode 100644 index 000000000000..f7fc5f8f1797 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -0,0 +1,230 @@ +//===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" + +namespace llvm { +namespace orc { + +CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) + : InitList( + GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), + I((InitList && End) ? InitList->getNumOperands() : 0) { +} + +bool CtorDtorIterator::operator==(const CtorDtorIterator &Other) const { + assert(InitList == Other.InitList && "Incomparable iterators."); + return I == Other.I; +} + +bool CtorDtorIterator::operator!=(const CtorDtorIterator &Other) const { + return !(*this == Other); +} + +CtorDtorIterator& CtorDtorIterator::operator++() { + ++I; + return *this; +} + +CtorDtorIterator CtorDtorIterator::operator++(int) { + CtorDtorIterator Temp = *this; + ++I; + return Temp; +} + +CtorDtorIterator::Element CtorDtorIterator::operator*() const { + ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(I)); + assert(CS && "Unrecognized type in llvm.global_ctors/llvm.global_dtors"); + + Constant *FuncC = CS->getOperand(1); + Function *Func = nullptr; + + // Extract function pointer, pulling off any casts. + while (FuncC) { + if (Function *F = dyn_cast_or_null<Function>(FuncC)) { + Func = F; + break; + } else if (ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(FuncC)) { + if (CE->isCast()) + FuncC = dyn_cast_or_null<ConstantExpr>(CE->getOperand(0)); + else + break; + } else { + // This isn't anything we recognize. Bail out with Func left set to null. + break; + } + } + + ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0)); + Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr; + if (Data && !isa<GlobalValue>(Data)) + Data = nullptr; + return Element(Priority->getZExtValue(), Func, Data); +} + +iterator_range<CtorDtorIterator> getConstructors(const Module &M) { + const GlobalVariable *CtorsList = M.getNamedGlobal("llvm.global_ctors"); + return make_range(CtorDtorIterator(CtorsList, false), + CtorDtorIterator(CtorsList, true)); +} + +iterator_range<CtorDtorIterator> getDestructors(const Module &M) { + const GlobalVariable *DtorsList = M.getNamedGlobal("llvm.global_dtors"); + return make_range(CtorDtorIterator(DtorsList, false), + CtorDtorIterator(DtorsList, true)); +} + +void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { + if (empty(CtorDtors)) + return; + + MangleAndInterner Mangle( + JD.getExecutionSession(), + (*CtorDtors.begin()).Func->getParent()->getDataLayout()); + + for (const auto &CtorDtor : CtorDtors) { + assert(CtorDtor.Func && CtorDtor.Func->hasName() && + "Ctor/Dtor function must be named to be runnable under the JIT"); + + // FIXME: Maybe use a symbol promoter here instead. + if (CtorDtor.Func->hasLocalLinkage()) { + CtorDtor.Func->setLinkage(GlobalValue::ExternalLinkage); + CtorDtor.Func->setVisibility(GlobalValue::HiddenVisibility); + } + + if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) { + dbgs() << " Skipping because why now?\n"; + continue; + } + + CtorDtorsByPriority[CtorDtor.Priority].push_back( + Mangle(CtorDtor.Func->getName())); + } +} + +Error CtorDtorRunner::run() { + using CtorDtorTy = void (*)(); + + SymbolNameSet Names; + + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + auto Added = Names.insert(Name).second; + (void)Added; + assert(Added && "Ctor/Dtor names clashed"); + } + } + + auto &ES = JD.getExecutionSession(); + if (auto CtorDtorMap = + ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names))) { + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + assert(CtorDtorMap->count(Name) && "No entry for Name"); + auto CtorDtor = reinterpret_cast<CtorDtorTy>( + static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + CtorDtor(); + } + } + CtorDtorsByPriority.clear(); + return Error::success(); + } else + return CtorDtorMap.takeError(); +} + +void LocalCXXRuntimeOverridesBase::runDestructors() { + auto& CXXDestructorDataPairs = DSOHandleOverride; + for (auto &P : CXXDestructorDataPairs) + P.first(P.second); + CXXDestructorDataPairs.clear(); +} + +int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, + void *Arg, + void *DSOHandle) { + auto& CXXDestructorDataPairs = + *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle); + CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg)); + return 0; +} + +Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, + MangleAndInterner &Mangle) { + SymbolMap RuntimeInterposes; + RuntimeInterposes[Mangle("__dso_handle")] = + JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), + JITSymbolFlags::Exported); + RuntimeInterposes[Mangle("__cxa_atexit")] = + JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), + JITSymbolFlags::Exported); + + return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); +} + +DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) + : Dylib(std::move(Dylib)), Allow(std::move(Allow)), + GlobalPrefix(GlobalPrefix) {} + +Expected<DynamicLibrarySearchGenerator> +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, + SymbolPredicate Allow) { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); + if (!Lib.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); +} + +Expected<SymbolNameSet> +DynamicLibrarySearchGenerator::operator()(JITDylib &JD, + const SymbolNameSet &Names) { + orc::SymbolNameSet Added; + orc::SymbolMap NewSymbols; + + bool HasGlobalPrefix = (GlobalPrefix != '\0'); + + for (auto &Name : Names) { + if ((*Name).empty()) + continue; + + if (Allow && !Allow(Name)) + continue; + + if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) + continue; + + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); + if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { + Added.insert(Name); + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), + JITSymbolFlags::Exported); + } + } + + // Add any new symbols to JD. Since the generator is only called for symbols + // that are not already defined, this will never trigger a duplicate + // definition error, so we can wrap this call in a 'cantFail'. + if (!NewSymbols.empty()) + cantFail(JD.define(absoluteSymbols(std::move(NewSymbols)))); + + return Added; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp new file mode 100644 index 000000000000..81dfc02f55b2 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -0,0 +1,43 @@ +//===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" + +namespace llvm { +namespace orc { + +IRCompileLayer::IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, + CompileFunction Compile) + : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {} + +void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + this->NotifyCompiled = std::move(NotifyCompiled); +} + +void IRCompileLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Module must not be null"); + + if (auto Obj = Compile(*TSM.getModule())) { + { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + if (NotifyCompiled) + NotifyCompiled(R.getVModuleKey(), std::move(TSM)); + else + TSM = ThreadSafeModule(); + } + BaseLayer.emit(std::move(R), std::move(*Obj)); + } else { + R.failMaterialization(); + getExecutionSession().reportError(Obj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp new file mode 100644 index 000000000000..e3519284613e --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -0,0 +1,33 @@ +//===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRTransformLayer::IRTransformLayer(ExecutionSession &ES, + IRLayer &BaseLayer, + TransformFunction Transform) + : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void IRTransformLayer::emit(MaterializationResponsibility R, + ThreadSafeModule TSM) { + assert(TSM.getModule() && "Module must not be null"); + + if (auto TransformedTSM = Transform(std::move(TSM), R)) + BaseLayer.emit(std::move(R), std::move(*TransformedTSM)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedTSM.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp new file mode 100644 index 000000000000..cc3656fe5dc5 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -0,0 +1,371 @@ +//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Support/Format.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include <sstream> + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { +public: + using CompileFunction = JITCompileCallbackManager::CompileFunction; + + CompileCallbackMaterializationUnit(SymbolStringPtr Name, + CompileFunction Compile, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), + std::move(K)), + Name(std::move(Name)), Compile(std::move(Compile)) {} + + StringRef getName() const override { return "<Compile Callbacks>"; } + +private: + void materialize(MaterializationResponsibility R) override { + SymbolMap Result; + Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + R.notifyResolved(Result); + R.notifyEmitted(); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { + llvm_unreachable("Discard should never occur on a LMU?"); + } + + SymbolStringPtr Name; + CompileFunction Compile; +}; + +} // namespace + +namespace llvm { +namespace orc { + +void IndirectStubsManager::anchor() {} +void TrampolinePool::anchor() {} + +Expected<JITTargetAddress> +JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { + if (auto TrampolineAddr = TP->getTrampoline()) { + auto CallbackName = + ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); + + std::lock_guard<std::mutex> Lock(CCMgrMutex); + AddrToSymbol[*TrampolineAddr] = CallbackName; + cantFail(CallbacksJD.define( + llvm::make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile), + ES.allocateVModule()))); + return *TrampolineAddr; + } else + return TrampolineAddr.takeError(); +} + +JITTargetAddress JITCompileCallbackManager::executeCompileCallback( + JITTargetAddress TrampolineAddr) { + SymbolStringPtr Name; + + { + std::unique_lock<std::mutex> Lock(CCMgrMutex); + auto I = AddrToSymbol.find(TrampolineAddr); + + // If this address is not associated with a compile callback then report an + // error to the execution session and return ErrorHandlerAddress to the + // callee. + if (I == AddrToSymbol.end()) { + Lock.unlock(); + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "No compile callback for trampoline at " + << format("0x%016" PRIx64, TrampolineAddr); + } + ES.reportError( + make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + return ErrorHandlerAddress; + } else + Name = I->second; + } + + if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name)) + return Sym->getAddress(); + else { + llvm::dbgs() << "Didn't find callback.\n"; + // If anything goes wrong materializing Sym then report it to the session + // and return the ErrorHandlerAddress; + ES.reportError(Sym.takeError()); + return ErrorHandlerAddress; + } +} + +Expected<std::unique_ptr<JITCompileCallbackManager>> +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) { + switch (T.getArch()) { + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + case Triple::aarch64: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::x86: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + case Triple::mipsel: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips64: + case Triple::mips64el: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::x86_64: { + if ( T.getOS() == Triple::OSType::Win32 ) { + typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } else { + typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + } + + } +} + +std::function<std::unique_ptr<IndirectStubsManager>()> +createLocalIndirectStubsManagerBuilder(const Triple &T) { + switch (T.getArch()) { + default: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); + }; + + case Triple::aarch64: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); + }; + + case Triple::x86: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcI386>>(); + }; + + case Triple::mips: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); + }; + + case Triple::mipsel: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); + }; + + case Triple::mips64: + case Triple::mips64el: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips64>>(); + }; + + case Triple::x86_64: + if (T.getOS() == Triple::OSType::Win32) { + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); + }; + } else { + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); + }; + } + + } +} + +Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { + Constant *AddrIntVal = + ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); + Constant *AddrPtrVal = + ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, + PointerType::get(&FT, 0)); + return AddrPtrVal; +} + +GlobalVariable* createImplPointer(PointerType &PT, Module &M, + const Twine &Name, Constant *Initializer) { + auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, + Initializer, Name, nullptr, + GlobalValue::NotThreadLocal, 0, true); + IP->setVisibility(GlobalValue::HiddenVisibility); + return IP; +} + +void makeStub(Function &F, Value &ImplPointer) { + assert(F.isDeclaration() && "Can't turn a definition into a stub."); + assert(F.getParent() && "Function isn't in a module."); + Module &M = *F.getParent(); + BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); + IRBuilder<> Builder(EntryBlock); + LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); + std::vector<Value*> CallArgs; + for (auto &A : F.args()) + CallArgs.push_back(&A); + CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); + Call->setTailCall(); + Call->setAttributes(F.getAttributes()); + if (F.getReturnType()->isVoidTy()) + Builder.CreateRetVoid(); + else + Builder.CreateRet(Call); +} + +std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { + std::vector<GlobalValue *> PromotedGlobals; + + for (auto &GV : M.global_values()) { + bool Promoted = true; + + // Rename if necessary. + if (!GV.hasName()) + GV.setName("__orc_anon." + Twine(NextId++)); + else if (GV.getName().startswith("\01L")) + GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); + else if (GV.hasLocalLinkage()) + GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); + else + Promoted = false; + + if (GV.hasLocalLinkage()) { + GV.setLinkage(GlobalValue::ExternalLinkage); + GV.setVisibility(GlobalValue::HiddenVisibility); + Promoted = true; + } + GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); + + if (Promoted) + PromotedGlobals.push_back(&GV); + } + + return PromotedGlobals; +} + +Function* cloneFunctionDecl(Module &Dst, const Function &F, + ValueToValueMapTy *VMap) { + Function *NewF = + Function::Create(cast<FunctionType>(F.getValueType()), + F.getLinkage(), F.getName(), &Dst); + NewF->copyAttributesFrom(&F); + + if (VMap) { + (*VMap)[&F] = NewF; + auto NewArgI = NewF->arg_begin(); + for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; + ++ArgI, ++NewArgI) + (*VMap)[&*ArgI] = &*NewArgI; + } + + return NewF; +} + +void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + Function *NewF) { + assert(!OrigF.isDeclaration() && "Nothing to move"); + if (!NewF) + NewF = cast<Function>(VMap[&OrigF]); + else + assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); + assert(NewF && "Function mapping missing from VMap."); + assert(NewF->getParent() != OrigF.getParent() && + "moveFunctionBody should only be used to move bodies between " + "modules."); + + SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. + CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, + "", nullptr, nullptr, Materializer); + OrigF.deleteBody(); +} + +GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, + ValueToValueMapTy *VMap) { + GlobalVariable *NewGV = new GlobalVariable( + Dst, GV.getValueType(), GV.isConstant(), + GV.getLinkage(), nullptr, GV.getName(), nullptr, + GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); + NewGV->copyAttributesFrom(&GV); + if (VMap) + (*VMap)[&GV] = NewGV; + return NewGV; +} + +void moveGlobalVariableInitializer(GlobalVariable &OrigGV, + ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + GlobalVariable *NewGV) { + assert(OrigGV.hasInitializer() && "Nothing to move"); + if (!NewGV) + NewGV = cast<GlobalVariable>(VMap[&OrigGV]); + else + assert(VMap[&OrigGV] == NewGV && + "Incorrect global variable mapping in VMap."); + assert(NewGV->getParent() != OrigGV.getParent() && + "moveGlobalVariableInitializer should only be used to move " + "initializers between modules"); + + NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, + nullptr, Materializer)); +} + +GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, + ValueToValueMapTy &VMap) { + assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); + auto *NewA = GlobalAlias::create(OrigA.getValueType(), + OrigA.getType()->getPointerAddressSpace(), + OrigA.getLinkage(), OrigA.getName(), &Dst); + NewA->copyAttributesFrom(&OrigA); + VMap[&OrigA] = NewA; + return NewA; +} + +void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, + ValueToValueMapTy &VMap) { + auto *MFs = Src.getModuleFlagsMetadata(); + if (!MFs) + return; + for (auto *MF : MFs->operands()) + Dst.addModuleFlag(MapMetadata(MF, VMap)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp new file mode 100644 index 000000000000..df23547a9de3 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -0,0 +1,54 @@ +//===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" + +#include "llvm/Support/TargetRegistry.h" + +namespace llvm { +namespace orc { + +JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) + : TT(std::move(TT)) { + Options.EmulatedTLS = true; + Options.ExplicitEmulatedTLS = true; +} + +Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { + // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, + // rather than a valid triple for the current process. + return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); +} + +Expected<std::unique_ptr<TargetMachine>> +JITTargetMachineBuilder::createTargetMachine() { + + std::string ErrMsg; + auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); + if (!TheTarget) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + auto *TM = + TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), + Options, RM, CM, OptLevel, /*JIT*/ true); + if (!TM) + return make_error<StringError>("Could not allocate target machine", + inconvertibleErrorCode()); + + return std::unique_ptr<TargetMachine>(TM); +} + +JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( + const std::vector<std::string> &FeatureVec) { + for (const auto &F : FeatureVec) + Features.AddFeature(F); + return *this; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp new file mode 100644 index 000000000000..b120691faf07 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -0,0 +1,226 @@ +//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/Mangler.h" + +namespace llvm { +namespace orc { + +Error LLJITBuilderState::prepareForConstruction() { + + if (!JTMB) { + if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) + JTMB = std::move(*JTMBOrErr); + else + return JTMBOrErr.takeError(); + } + + return Error::success(); +} + +LLJIT::~LLJIT() { + if (CompileThreads) + CompileThreads->wait(); +} + +Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { + auto InternedName = ES->intern(Name); + SymbolMap Symbols({{InternedName, Sym}}); + return Main.define(absoluteSymbols(std::move(Symbols))); +} + +Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); + + if (auto Err = applyDataLayout(*TSM.getModule())) + return Err; + + return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); +} + +Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { + assert(Obj && "Can not add null object"); + + return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule()); +} + +Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, + StringRef Name) { + return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); +} + +std::unique_ptr<ObjectLayer> +LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { + + // If the config state provided an ObjectLinkingLayer factory then use it. + if (S.CreateObjectLinkingLayer) + return S.CreateObjectLinkingLayer(ES); + + // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs + // a new SectionMemoryManager for each object. + auto GetMemMgr = []() { return llvm::make_unique<SectionMemoryManager>(); }; + return llvm::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); +} + +Expected<IRCompileLayer::CompileFunction> +LLJIT::createCompileFunction(LLJITBuilderState &S, + JITTargetMachineBuilder JTMB) { + + /// If there is a custom compile function creator set then use it. + if (S.CreateCompileFunction) + return S.CreateCompileFunction(std::move(JTMB)); + + // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, + // depending on the number of threads requested. + if (S.NumCompileThreads > 0) + return ConcurrentIRCompiler(std::move(JTMB)); + + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + + return TMOwningSimpleCompiler(std::move(*TM)); +} + +LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) + : ES(S.ES ? std::move(S.ES) : llvm::make_unique<ExecutionSession>()), + Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), + DtorRunner(Main) { + + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer = createObjectLinkingLayer(S, *ES); + + if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else { + Err = DLOrErr.takeError(); + return; + } + + { + auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); + if (!CompileFunction) { + Err = CompileFunction.takeError(); + return; + } + CompileLayer = llvm::make_unique<IRCompileLayer>( + *ES, *ObjLinkingLayer, std::move(*CompileFunction)); + } + + if (S.NumCompileThreads > 0) { + CompileLayer->setCloneToNewContextOnEmit(true); + CompileThreads = llvm::make_unique<ThreadPool>(S.NumCompileThreads); + ES->setDispatchMaterialization( + [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { + // FIXME: Switch to move capture once we have c++14. + auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); + auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + CompileThreads->async(std::move(Work)); + }); + } +} + +std::string LLJIT::mangle(StringRef UnmangledName) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL); + } + return MangledName; +} + +Error LLJIT::applyDataLayout(Module &M) { + if (M.getDataLayout().isDefault()) + M.setDataLayout(DL); + + if (M.getDataLayout() != DL) + return make_error<StringError>( + "Added modules have incompatible data layouts", + inconvertibleErrorCode()); + + return Error::success(); +} + +void LLJIT::recordCtorDtors(Module &M) { + CtorRunner.add(getConstructors(M)); + DtorRunner.add(getDestructors(M)); +} + +Error LLLazyJITBuilderState::prepareForConstruction() { + if (auto Err = LLJITBuilderState::prepareForConstruction()) + return Err; + TT = JTMB->getTargetTriple(); + return Error::success(); +} + +Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); + + if (auto Err = applyDataLayout(*TSM.getModule())) + return Err; + + recordCtorDtors(*TSM.getModule()); + + return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); +} + +LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { + + // If LLJIT construction failed then bail out. + if (Err) + return; + + ErrorAsOutParameter _(&Err); + + /// Take/Create the lazy-compile callthrough manager. + if (S.LCTMgr) + LCTMgr = std::move(S.LCTMgr); + else { + if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( + S.TT, *ES, S.LazyCompileFailureAddr)) + LCTMgr = std::move(*LCTMgrOrErr); + else { + Err = LCTMgrOrErr.takeError(); + return; + } + } + + // Take/Create the indirect stubs manager builder. + auto ISMBuilder = std::move(S.ISMBuilder); + + // If none was provided, try to build one. + if (!ISMBuilder) + ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); + + // No luck. Bail out. + if (!ISMBuilder) { + Err = make_error<StringError>("Could not construct " + "IndirectStubsManagerBuilder for target " + + S.TT.str(), + inconvertibleErrorCode()); + return; + } + + // Create the transform layer. + TransformLayer = llvm::make_unique<IRTransformLayer>(*ES, *CompileLayer); + + // Create the COD layer. + CODLayer = llvm::make_unique<CompileOnDemandLayer>( + *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + + if (S.NumCompileThreads > 0) + CODLayer->setCloneToNewContextOnEmit(true); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp new file mode 100644 index 000000000000..3ed2dabf4545 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -0,0 +1,183 @@ +//===-------------------- Layer.cpp - Layer interfaces --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} +IRLayer::~IRLayer() {} + +Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { + return JD.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( + *this, std::move(K), std::move(TSM))); +} + +IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, + ThreadSafeModule TSM, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap(), std::move(K)), TSM(std::move(TSM)) { + + assert(this->TSM && "Module must not be null"); + + MangleAndInterner Mangle(ES, this->TSM.getModule()->getDataLayout()); + for (auto &G : this->TSM.getModule()->global_values()) { + if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && + !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } + } +} + +IRMaterializationUnit::IRMaterializationUnit( + ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags), std::move(K)), + TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} + +StringRef IRMaterializationUnit::getName() const { + if (TSM.getModule()) + return TSM.getModule()->getModuleIdentifier(); + return "<null module>"; +} + +void IRMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD.getName() << " discarding " << *Name << " from MU@" + << this << " (" << getName() << ")\n"; + });); + + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && + "Symbol not provided by this MU, or previously discarded"); + assert(!I->second->isDeclaration() && + "Discard should only apply to definitions"); + I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + SymbolToDefinition.erase(I); +} + +BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( + IRLayer &L, VModuleKey K, ThreadSafeModule TSM) + : IRMaterializationUnit(L.getExecutionSession(), std::move(TSM), + std::move(K)), + L(L), K(std::move(K)) {} + +void BasicIRLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + + // Throw away the SymbolToDefinition map: it's not usable after we hand + // off the module. + SymbolToDefinition.clear(); + + // If cloneToNewContextOnEmit is set, clone the module now. + if (L.getCloneToNewContextOnEmit()) + TSM = cloneToNewContext(TSM); + +#ifndef NDEBUG + auto &ES = R.getTargetJITDylib().getExecutionSession(); + auto &N = R.getTargetJITDylib().getName(); +#endif // NDEBUG + + auto Lock = TSM.getContextLock(); + LLVM_DEBUG(ES.runSessionLocked( + [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); + L.emit(std::move(R), std::move(TSM)); + LLVM_DEBUG(ES.runSessionLocked([&]() { + dbgs() << "Finished emitting, for " << N << ", " << *this << "\n"; + });); +} + +ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} + +ObjectLayer::~ObjectLayer() {} + +Error ObjectLayer::add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O, + VModuleKey K) { + auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(K), + std::move(O)); + if (!ObjMU) + return ObjMU.takeError(); + return JD.define(std::move(*ObjMU)); +} + +Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> +BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + auto SymbolFlags = + getObjectSymbolFlags(L.getExecutionSession(), O->getMemBufferRef()); + + if (!SymbolFlags) + return SymbolFlags.takeError(); + + return std::unique_ptr<BasicObjectLayerMaterializationUnit>( + new BasicObjectLayerMaterializationUnit(L, K, std::move(O), + std::move(*SymbolFlags))); +} + +BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( + ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O, + SymbolFlagsMap SymbolFlags) + : MaterializationUnit(std::move(SymbolFlags), std::move(K)), L(L), + O(std::move(O)) {} + +StringRef BasicObjectLayerMaterializationUnit::getName() const { + if (O) + return O->getBufferIdentifier(); + return "<null object>"; +} + +void BasicObjectLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + L.emit(std::move(R), std::move(O)); +} + +void BasicObjectLayerMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + // FIXME: Support object file level discard. This could be done by building a + // filter to pass to the object layer along with the object itself. +} + +Expected<SymbolFlagsMap> getObjectSymbolFlags(ExecutionSession &ES, + MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + // Skip symbols not defined in this object file. + if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) + continue; + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + return SymbolFlags; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp new file mode 100644 index 000000000000..fc8205845654 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -0,0 +1,204 @@ +//===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +void LazyCallThroughManager::NotifyResolvedFunction::anchor() {} + +LazyCallThroughManager::LazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, + std::unique_ptr<TrampolinePool> TP) + : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(std::move(TP)) {} + +Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( + JITDylib &SourceJD, SymbolStringPtr SymbolName, + std::shared_ptr<NotifyResolvedFunction> NotifyResolved) { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto Trampoline = TP->getTrampoline(); + + if (!Trampoline) + return Trampoline.takeError(); + + Reexports[*Trampoline] = std::make_pair(&SourceJD, std::move(SymbolName)); + Notifiers[*Trampoline] = std::move(NotifyResolved); + return *Trampoline; +} + +JITTargetAddress +LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { + JITDylib *SourceJD = nullptr; + SymbolStringPtr SymbolName; + + { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Reexports.find(TrampolineAddr); + if (I == Reexports.end()) + return ErrorHandlerAddr; + SourceJD = I->second.first; + SymbolName = I->second.second; + } + + auto LookupResult = + ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); + + if (!LookupResult) { + ES.reportError(LookupResult.takeError()); + return ErrorHandlerAddr; + } + + auto ResolvedAddr = LookupResult->getAddress(); + + std::shared_ptr<NotifyResolvedFunction> NotifyResolved = nullptr; + { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Notifiers.find(TrampolineAddr); + if (I != Notifiers.end()) { + NotifyResolved = I->second; + Notifiers.erase(I); + } + } + + if (NotifyResolved) { + if (auto Err = (*NotifyResolved)(*SourceJD, SymbolName, ResolvedAddr)) { + ES.reportError(std::move(Err)); + return ErrorHandlerAddr; + } + } + + return ResolvedAddr; +} + +Expected<std::unique_ptr<LazyCallThroughManager>> +createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr) { + switch (T.getArch()) { + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + + case Triple::aarch64: + return LocalLazyCallThroughManager::Create<OrcAArch64>(ES, + ErrorHandlerAddr); + + case Triple::x86: + return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr); + + case Triple::mips: + return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES, + ErrorHandlerAddr); + + case Triple::mipsel: + return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES, + ErrorHandlerAddr); + + case Triple::mips64: + case Triple::mips64el: + return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr); + + case Triple::x86_64: + if (T.getOS() == Triple::OSType::Win32) + return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>( + ES, ErrorHandlerAddr); + else + return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>( + ES, ErrorHandlerAddr); + } +} + +LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( + LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, + JITDylib &SourceJD, SymbolAliasMap CallableAliases, VModuleKey K) + : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), + LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), + CallableAliases(std::move(CallableAliases)), + NotifyResolved(LazyCallThroughManager::createNotifyResolvedFunction( + [&ISManager](JITDylib &JD, const SymbolStringPtr &SymbolName, + JITTargetAddress ResolvedAddr) { + return ISManager.updatePointer(*SymbolName, ResolvedAddr); + })) {} + +StringRef LazyReexportsMaterializationUnit::getName() const { + return "<Lazy Reexports>"; +} + +void LazyReexportsMaterializationUnit::materialize( + MaterializationResponsibility R) { + auto RequestedSymbols = R.getRequestedSymbols(); + + SymbolAliasMap RequestedAliases; + for (auto &RequestedSymbol : RequestedSymbols) { + auto I = CallableAliases.find(RequestedSymbol); + assert(I != CallableAliases.end() && "Symbol not found in alias map?"); + RequestedAliases[I->first] = std::move(I->second); + CallableAliases.erase(I); + } + + if (!CallableAliases.empty()) + R.replace(lazyReexports(LCTManager, ISManager, SourceJD, + std::move(CallableAliases))); + + IndirectStubsManager::StubInitsMap StubInits; + for (auto &Alias : RequestedAliases) { + + auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( + SourceJD, Alias.second.Aliasee, NotifyResolved); + + if (!CallThroughTrampoline) { + SourceJD.getExecutionSession().reportError( + CallThroughTrampoline.takeError()); + R.failMaterialization(); + return; + } + + StubInits[*Alias.first] = + std::make_pair(*CallThroughTrampoline, Alias.second.AliasFlags); + } + + if (auto Err = ISManager.createStubs(StubInits)) { + SourceJD.getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + SymbolMap Stubs; + for (auto &Alias : RequestedAliases) + Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); + + R.notifyResolved(Stubs); + R.notifyEmitted(); +} + +void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(CallableAliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + CallableAliases.erase(Name); +} + +SymbolFlagsMap +LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) { + assert(KV.second.AliasFlags.isCallable() && + "Lazy re-exports must be callable symbols"); + SymbolFlags[KV.first] = KV.second.AliasFlags; + } + return SymbolFlags; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp new file mode 100644 index 000000000000..ce6368b57a89 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp @@ -0,0 +1,66 @@ +//===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Legacy.h" + +namespace llvm { +namespace orc { + +void SymbolResolver::anchor() {} + +JITSymbolResolverAdapter::JITSymbolResolverAdapter( + ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR) + : ES(ES), R(R), MR(MR) {} + +void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, + OnResolvedFunction OnResolved) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + auto OnResolvedWithUnwrap = [OnResolved](Expected<SymbolMap> InternedResult) { + if (!InternedResult) { + OnResolved(InternedResult.takeError()); + return; + } + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + OnResolved(Result); + }; + + auto Q = std::make_shared<AsynchronousSymbolQuery>( + InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap); + + auto Unresolved = R.lookup(Q, InternedSymbols); + if (Unresolved.empty()) { + if (MR) + MR->addDependenciesForAll(Q->QueryRegistrations); + } else + ES.legacyFailQuery(*Q, make_error<SymbolsNotFound>(std::move(Unresolved))); +} + +Expected<JITSymbolResolverAdapter::LookupSet> +JITSymbolResolverAdapter::getResponsibilitySet(const LookupSet &Symbols) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + auto InternedResult = R.getResponsibilitySet(InternedSymbols); + LookupSet Result; + for (auto &S : InternedResult) { + ResolvedStrings.insert(S); + Result.insert(*S); + } + + return Result; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp new file mode 100644 index 000000000000..5b4345b870bb --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -0,0 +1,37 @@ +//===---------- NullResolver.cpp - Reject symbol lookup requests ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/NullResolver.h" + +#include "llvm/Support/ErrorHandling.h" + +namespace llvm { +namespace orc { + +SymbolNameSet NullResolver::getResponsibilitySet(const SymbolNameSet &Symbols) { + return Symbols; +} + +SymbolNameSet +NullResolver::lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) { + assert(Symbols.empty() && "Null resolver: Symbols must be empty"); + return Symbols; +} + +JITSymbol NullLegacyResolver::findSymbol(const std::string &Name) { + llvm_unreachable("Unexpected cross-object symbol reference"); +} + +JITSymbol +NullLegacyResolver::findSymbolInLogicalDylib(const std::string &Name) { + llvm_unreachable("Unexpected cross-object symbol reference"); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 000000000000..def0b300eca1 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,483 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include <vector> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet<StringRef> &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared<JITLinkAsyncLookupContinuation>( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved, + std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.notifyResolved(InternedResult); + + Layer.notifyLoaded(MR); + } + + void notifyFinalized( + std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + + if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + + return; + } + MR.notifyEmitted(); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Layer.modifyPassConfig(MR, TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap<const DefinedAtom *, SymbolNameSet>; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast<DefinedAtom &>(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet<DefinedAtom *> DAAnonDeps; + }; + std::vector<WorklistEntry> Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet<DefinedAtom *> DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr<MemoryBuffer> ObjBuffer; + DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps; +}; + +ObjectLinkingLayer::Plugin::~Plugin() {} + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) {} + +ObjectLinkingLayer::~ObjectLinkingLayer() { + if (auto Err = removeAllModules()) + getExecutionSession().reportError(std::move(Err)); +} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O))); +} + +void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + PassConfiguration &PassConfig) { + for (auto &P : Plugins) + P->modifyPassConfig(MR, TT, PassConfig); +} + +void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { + for (auto &P : Plugins) + P->notifyLoaded(MR); +} + +Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, + AllocPtr Alloc) { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); + + if (Err) + return Err; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + UntrackedAllocs.push_back(std::move(Alloc)); + } + + return Error::success(); +} + +Error ObjectLinkingLayer::removeModule(VModuleKey K) { + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingModule(K)); + + AllocPtr Alloc; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + auto AllocItr = TrackedAllocs.find(K); + Alloc = std::move(AllocItr->second); + TrackedAllocs.erase(AllocItr); + } + + assert(Alloc && "No allocation for key K"); + + return joinErrors(std::move(Err), Alloc->deallocate()); +} + +Error ObjectLinkingLayer::removeAllModules() { + + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingAllModules()); + + std::vector<AllocPtr> Allocs; + { + std::lock_guard<std::mutex> Lock(LayerMutex); + Allocs = std::move(UntrackedAllocs); + + for (auto &KV : TrackedAllocs) + Allocs.push_back(std::move(KV.second)); + + TrackedAllocs.clear(); + } + + while (!Allocs.empty()) { + Err = joinErrors(std::move(Err), Allocs.back()->deallocate()); + Allocs.pop_back(); + } + + return Err; +} + +EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( + jitlink::EHFrameRegistrar &Registrar) + : Registrar(Registrar) {} + +void EHFrameRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + PassConfiguration &PassConfig) { + assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); + + PassConfig.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) { + if (Addr) + InProcessLinks[&MR] = Addr; + })); +} + +Error EHFrameRegistrationPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + + auto EHFrameAddrItr = InProcessLinks.find(&MR); + if (EHFrameAddrItr == InProcessLinks.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "eh-frame addr to register can not be null"); + + InProcessLinks.erase(EHFrameAddrItr); + if (auto Key = MR.getVModuleKey()) + TrackedEHFrameAddrs[Key] = EHFrameAddr; + else + UntrackedEHFrameAddrs.push_back(EHFrameAddr); + + return Registrar.registerEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { + auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K); + if (EHFrameAddrItr == TrackedEHFrameAddrs.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "Tracked eh-frame addr must not be null"); + + TrackedEHFrameAddrs.erase(EHFrameAddrItr); + + return Registrar.deregisterEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { + + std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs); + EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size()); + + for (auto &KV : TrackedEHFrameAddrs) + EHFrameAddrs.push_back(KV.second); + + TrackedEHFrameAddrs.clear(); + + Error Err = Error::success(); + + while (!EHFrameAddrs.empty()) { + auto EHFrameAddr = EHFrameAddrs.back(); + assert(EHFrameAddr && "Untracked eh-frame addr must not be null"); + EHFrameAddrs.pop_back(); + Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr)); + } + + return Err; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp new file mode 100644 index 000000000000..815517321b76 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -0,0 +1,33 @@ +//===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +ObjectTransformLayer::ObjectTransformLayer(ExecutionSession &ES, + ObjectLayer &BaseLayer, + TransformFunction Transform) + : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void ObjectTransformLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Module must not be null"); + + if (auto TransformedObj = Transform(std::move(O))) + BaseLayer.emit(std::move(R), std::move(*TransformedObj)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedObj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp new file mode 100644 index 000000000000..8ed23de419d1 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -0,0 +1,983 @@ +//===------------- OrcABISupport.cpp - ABI specific support code ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/Support/Process.h" + +namespace llvm { +namespace orc { + +void OrcAArch64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint32_t ResolverCode[] = { + // resolver_entry: + 0xa9bf47fd, // 0x000: stp x29, x17, [sp, #-16]! + 0x910003fd, // 0x004: mov x29, sp + 0xa9bf73fb, // 0x008: stp x27, x28, [sp, #-16]! + 0xa9bf6bf9, // 0x00c: stp x25, x26, [sp, #-16]! + 0xa9bf63f7, // 0x010: stp x23, x24, [sp, #-16]! + 0xa9bf5bf5, // 0x014: stp x21, x22, [sp, #-16]! + 0xa9bf53f3, // 0x018: stp x19, x20, [sp, #-16]! + 0xa9bf3fee, // 0x01c: stp x14, x15, [sp, #-16]! + 0xa9bf37ec, // 0x020: stp x12, x13, [sp, #-16]! + 0xa9bf2fea, // 0x024: stp x10, x11, [sp, #-16]! + 0xa9bf27e8, // 0x028: stp x8, x9, [sp, #-16]! + 0xa9bf1fe6, // 0x02c: stp x6, x7, [sp, #-16]! + 0xa9bf17e4, // 0x030: stp x4, x5, [sp, #-16]! + 0xa9bf0fe2, // 0x034: stp x2, x3, [sp, #-16]! + 0xa9bf07e0, // 0x038: stp x0, x1, [sp, #-16]! + 0xadbf7ffe, // 0x03c: stp q30, q31, [sp, #-32]! + 0xadbf77fc, // 0x040: stp q28, q29, [sp, #-32]! + 0xadbf6ffa, // 0x044: stp q26, q27, [sp, #-32]! + 0xadbf67f8, // 0x048: stp q24, q25, [sp, #-32]! + 0xadbf5ff6, // 0x04c: stp q22, q23, [sp, #-32]! + 0xadbf57f4, // 0x050: stp q20, q21, [sp, #-32]! + 0xadbf4ff2, // 0x054: stp q18, q19, [sp, #-32]! + 0xadbf47f0, // 0x058: stp q16, q17, [sp, #-32]! + 0xadbf3fee, // 0x05c: stp q14, q15, [sp, #-32]! + 0xadbf37ec, // 0x060: stp q12, q13, [sp, #-32]! + 0xadbf2fea, // 0x064: stp q10, q11, [sp, #-32]! + 0xadbf27e8, // 0x068: stp q8, q9, [sp, #-32]! + 0xadbf1fe6, // 0x06c: stp q6, q7, [sp, #-32]! + 0xadbf17e4, // 0x070: stp q4, q5, [sp, #-32]! + 0xadbf0fe2, // 0x074: stp q2, q3, [sp, #-32]! + 0xadbf07e0, // 0x078: stp q0, q1, [sp, #-32]! + 0x580004e0, // 0x07c: ldr x0, Lcallbackmgr + 0xaa1e03e1, // 0x080: mov x1, x30 + 0xd1003021, // 0x084: sub x1, x1, #12 + 0x58000442, // 0x088: ldr x2, Lreentry_fn_ptr + 0xd63f0040, // 0x08c: blr x2 + 0xaa0003f1, // 0x090: mov x17, x0 + 0xacc107e0, // 0x094: ldp q0, q1, [sp], #32 + 0xacc10fe2, // 0x098: ldp q2, q3, [sp], #32 + 0xacc117e4, // 0x09c: ldp q4, q5, [sp], #32 + 0xacc11fe6, // 0x0a0: ldp q6, q7, [sp], #32 + 0xacc127e8, // 0x0a4: ldp q8, q9, [sp], #32 + 0xacc12fea, // 0x0a8: ldp q10, q11, [sp], #32 + 0xacc137ec, // 0x0ac: ldp q12, q13, [sp], #32 + 0xacc13fee, // 0x0b0: ldp q14, q15, [sp], #32 + 0xacc147f0, // 0x0b4: ldp q16, q17, [sp], #32 + 0xacc14ff2, // 0x0b8: ldp q18, q19, [sp], #32 + 0xacc157f4, // 0x0bc: ldp q20, q21, [sp], #32 + 0xacc15ff6, // 0x0c0: ldp q22, q23, [sp], #32 + 0xacc167f8, // 0x0c4: ldp q24, q25, [sp], #32 + 0xacc16ffa, // 0x0c8: ldp q26, q27, [sp], #32 + 0xacc177fc, // 0x0cc: ldp q28, q29, [sp], #32 + 0xacc17ffe, // 0x0d0: ldp q30, q31, [sp], #32 + 0xa8c107e0, // 0x0d4: ldp x0, x1, [sp], #16 + 0xa8c10fe2, // 0x0d8: ldp x2, x3, [sp], #16 + 0xa8c117e4, // 0x0dc: ldp x4, x5, [sp], #16 + 0xa8c11fe6, // 0x0e0: ldp x6, x7, [sp], #16 + 0xa8c127e8, // 0x0e4: ldp x8, x9, [sp], #16 + 0xa8c12fea, // 0x0e8: ldp x10, x11, [sp], #16 + 0xa8c137ec, // 0x0ec: ldp x12, x13, [sp], #16 + 0xa8c13fee, // 0x0f0: ldp x14, x15, [sp], #16 + 0xa8c153f3, // 0x0f4: ldp x19, x20, [sp], #16 + 0xa8c15bf5, // 0x0f8: ldp x21, x22, [sp], #16 + 0xa8c163f7, // 0x0fc: ldp x23, x24, [sp], #16 + 0xa8c16bf9, // 0x100: ldp x25, x26, [sp], #16 + 0xa8c173fb, // 0x104: ldp x27, x28, [sp], #16 + 0xa8c17bfd, // 0x108: ldp x29, x30, [sp], #16 + 0xd65f0220, // 0x10c: ret x17 + 0x01234567, // 0x110: Lreentry_fn_ptr: + 0xdeadbeef, // 0x114: .quad 0 + 0x98765432, // 0x118: Lcallbackmgr: + 0xcafef00d // 0x11c: .quad 0 + }; + + const unsigned ReentryFnAddrOffset = 0x110; + const unsigned CallbackMgrAddrOffset = 0x118; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, + sizeof(CallbackMgr)); +} + +void OrcAArch64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { + + unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); + + memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); + + // OffsetToPtr is actually the offset from the PC for the 2nd instruction, so + // subtract 32-bits. + OffsetToPtr -= 4; + + uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) { + Trampolines[3 * I + 0] = 0xaa1e03f1; // mov x17, x30 + Trampolines[3 * I + 1] = 0x58000010 | (OffsetToPtr << 3); // adr x16, Lptr + Trampolines[3 * I + 2] = 0xd63f0200; // blr x16 + } + +} + +Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // ldr x0, ptr1 ; PC-rel load of ptr1 + // br x0 ; Jump to resolver + // stub2: + // ldr x0, ptr2 ; PC-rel load of ptr2 + // br x0 ; Jump to resolver + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); + uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize) + << 3; + + for (unsigned I = 0; I < NumStubs; ++I) + Stub[I] = 0xd61f020058000010 | PtrOffsetField; + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} + +void OrcX86_64_Base::writeTrampolines(uint8_t *TrampolineMem, + void *ResolverAddr, + unsigned NumTrampolines) { + + unsigned OffsetToPtr = NumTrampolines * TrampolineSize; + + memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); + + uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); + uint64_t CallIndirPCRel = 0xf1c40000000015ff; + + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) + Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); +} + +Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // jmpq *ptr1(%rip) + // .byte 0xC4 ; <- Invalid opcode padding. + // .byte 0xF1 + // stub2: + // jmpq *ptr2(%rip) + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); + uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize - 6) + << 16; + for (unsigned I = 0; I < NumStubs; ++I) + Stub[I] = 0xF1C40000000025ff | PtrOffsetField; + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} + +void OrcX86_64_SysV::writeResolverCode(uint8_t *ResolverMem, + JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushq %rbp + 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp + 0x50, // 0x04: pushq %rax + 0x53, // 0x05: pushq %rbx + 0x51, // 0x06: pushq %rcx + 0x52, // 0x07: pushq %rdx + 0x56, // 0x08: pushq %rsi + 0x57, // 0x09: pushq %rdi + 0x41, 0x50, // 0x0a: pushq %r8 + 0x41, 0x51, // 0x0c: pushq %r9 + 0x41, 0x52, // 0x0e: pushq %r10 + 0x41, 0x53, // 0x10: pushq %r11 + 0x41, 0x54, // 0x12: pushq %r12 + 0x41, 0x55, // 0x14: pushq %r13 + 0x41, 0x56, // 0x16: pushq %r14 + 0x41, 0x57, // 0x18: pushq %r15 + 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp + 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) + 0x48, 0xbf, // 0x26: movabsq <CBMgr>, %rdi + + // 0x28: Callback manager addr. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi + 0x48, 0x83, 0xee, 0x06, // 0x34: subq $6, %rsi + 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax + + // 0x3a: JIT re-entry fn addr: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0xff, 0xd0, // 0x42: callq *%rax + 0x48, 0x89, 0x45, 0x08, // 0x44: movq %rax, 8(%rbp) + 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x48: fxrstor64 (%rsp) + 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x4d: addq 0x208, %rsp + 0x41, 0x5f, // 0x54: popq %r15 + 0x41, 0x5e, // 0x56: popq %r14 + 0x41, 0x5d, // 0x58: popq %r13 + 0x41, 0x5c, // 0x5a: popq %r12 + 0x41, 0x5b, // 0x5c: popq %r11 + 0x41, 0x5a, // 0x5e: popq %r10 + 0x41, 0x59, // 0x60: popq %r9 + 0x41, 0x58, // 0x62: popq %r8 + 0x5f, // 0x64: popq %rdi + 0x5e, // 0x65: popq %rsi + 0x5a, // 0x66: popq %rdx + 0x59, // 0x67: popq %rcx + 0x5b, // 0x68: popq %rbx + 0x58, // 0x69: popq %rax + 0x5d, // 0x6a: popq %rbp + 0xc3, // 0x6b: retq + }; + + const unsigned ReentryFnAddrOffset = 0x3a; + const unsigned CallbackMgrAddrOffset = 0x28; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, + sizeof(CallbackMgr)); +} + +void OrcX86_64_Win32::writeResolverCode(uint8_t *ResolverMem, + JITReentryFn ReentryFn, + void *CallbackMgr) { + + // resolverCode is similar to OrcX86_64 with differences specific to windows x64 calling convention: + // arguments go into rcx, rdx and come in reverse order, shadow space allocation on stack + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushq %rbp + 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp + 0x50, // 0x04: pushq %rax + 0x53, // 0x05: pushq %rbx + 0x51, // 0x06: pushq %rcx + 0x52, // 0x07: pushq %rdx + 0x56, // 0x08: pushq %rsi + 0x57, // 0x09: pushq %rdi + 0x41, 0x50, // 0x0a: pushq %r8 + 0x41, 0x51, // 0x0c: pushq %r9 + 0x41, 0x52, // 0x0e: pushq %r10 + 0x41, 0x53, // 0x10: pushq %r11 + 0x41, 0x54, // 0x12: pushq %r12 + 0x41, 0x55, // 0x14: pushq %r13 + 0x41, 0x56, // 0x16: pushq %r14 + 0x41, 0x57, // 0x18: pushq %r15 + 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp + 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) + + 0x48, 0xb9, // 0x26: movabsq <CBMgr>, %rcx + // 0x28: Callback manager addr. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0x48, 0x8B, 0x55, 0x08, // 0x30: mov rdx, [rbp+0x8] + 0x48, 0x83, 0xea, 0x06, // 0x34: sub rdx, 0x6 + + 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax + // 0x3a: JIT re-entry fn addr: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // 0x42: sub rsp, 0x20 (Allocate shadow space) + 0x48, 0x83, 0xEC, 0x20, + 0xff, 0xd0, // 0x46: callq *%rax + + // 0x48: add rsp, 0x20 (Free shadow space) + 0x48, 0x83, 0xC4, 0x20, + + 0x48, 0x89, 0x45, 0x08, // 0x4C: movq %rax, 8(%rbp) + 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x50: fxrstor64 (%rsp) + 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x55: addq 0x208, %rsp + 0x41, 0x5f, // 0x5C: popq %r15 + 0x41, 0x5e, // 0x5E: popq %r14 + 0x41, 0x5d, // 0x60: popq %r13 + 0x41, 0x5c, // 0x62: popq %r12 + 0x41, 0x5b, // 0x64: popq %r11 + 0x41, 0x5a, // 0x66: popq %r10 + 0x41, 0x59, // 0x68: popq %r9 + 0x41, 0x58, // 0x6a: popq %r8 + 0x5f, // 0x6c: popq %rdi + 0x5e, // 0x6d: popq %rsi + 0x5a, // 0x6e: popq %rdx + 0x59, // 0x6f: popq %rcx + 0x5b, // 0x70: popq %rbx + 0x58, // 0x71: popq %rax + 0x5d, // 0x72: popq %rbp + 0xc3, // 0x73: retq + }; + + + const unsigned ReentryFnAddrOffset = 0x3a; + const unsigned CallbackMgrAddrOffset = 0x28; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, + sizeof(CallbackMgr)); +} + +void OrcI386::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushl %ebp + 0x89, 0xe5, // 0x01: movl %esp, %ebp + 0x54, // 0x03: pushl %esp + 0x83, 0xe4, 0xf0, // 0x04: andl $-0x10, %esp + 0x50, // 0x07: pushl %eax + 0x53, // 0x08: pushl %ebx + 0x51, // 0x09: pushl %ecx + 0x52, // 0x0a: pushl %edx + 0x56, // 0x0b: pushl %esi + 0x57, // 0x0c: pushl %edi + 0x81, 0xec, 0x18, 0x02, 0x00, 0x00, // 0x0d: subl $0x218, %esp + 0x0f, 0xae, 0x44, 0x24, 0x10, // 0x13: fxsave 0x10(%esp) + 0x8b, 0x75, 0x04, // 0x18: movl 0x4(%ebp), %esi + 0x83, 0xee, 0x05, // 0x1b: subl $0x5, %esi + 0x89, 0x74, 0x24, 0x04, // 0x1e: movl %esi, 0x4(%esp) + 0xc7, 0x04, 0x24, 0x00, 0x00, 0x00, + 0x00, // 0x22: movl <cbmgr>, (%esp) + 0xb8, 0x00, 0x00, 0x00, 0x00, // 0x29: movl <reentry>, %eax + 0xff, 0xd0, // 0x2e: calll *%eax + 0x89, 0x45, 0x04, // 0x30: movl %eax, 0x4(%ebp) + 0x0f, 0xae, 0x4c, 0x24, 0x10, // 0x33: fxrstor 0x10(%esp) + 0x81, 0xc4, 0x18, 0x02, 0x00, 0x00, // 0x38: addl $0x218, %esp + 0x5f, // 0x3e: popl %edi + 0x5e, // 0x3f: popl %esi + 0x5a, // 0x40: popl %edx + 0x59, // 0x41: popl %ecx + 0x5b, // 0x42: popl %ebx + 0x58, // 0x43: popl %eax + 0x8b, 0x65, 0xfc, // 0x44: movl -0x4(%ebp), %esp + 0x5d, // 0x48: popl %ebp + 0xc3 // 0x49: retl + }; + + const unsigned ReentryFnAddrOffset = 0x2a; + const unsigned CallbackMgrAddrOffset = 0x25; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, + sizeof(CallbackMgr)); +} + +void OrcI386::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { + + uint64_t CallRelImm = 0xF1C4C400000000e8; + uint64_t Resolver = reinterpret_cast<uint64_t>(ResolverAddr); + uint64_t ResolverRel = + Resolver - reinterpret_cast<uint64_t>(TrampolineMem) - 5; + + uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); + for (unsigned I = 0; I < NumTrampolines; ++I, ResolverRel -= TrampolineSize) + Trampolines[I] = CallRelImm | (ResolverRel << 8); +} + +Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // jmpq *ptr1 + // .byte 0xC4 ; <- Invalid opcode padding. + // .byte 0xF1 + // stub2: + // jmpq *ptr2 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); + uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) + Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} + +void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, + JITReentryFn ReentryFn, + void *CallbackMgr, bool isBigEndian) { + + const uint32_t ResolverCode[] = { + // resolver_entry: + 0x27bdff98, // 0x00: addiu $sp,$sp,-104 + 0xafa20000, // 0x04: sw $v0,0($sp) + 0xafa30004, // 0x08: sw $v1,4($sp) + 0xafa40008, // 0x0c: sw $a0,8($sp) + 0xafa5000c, // 0x10: sw $a1,12($sp) + 0xafa60010, // 0x14: sw $a2,16($sp) + 0xafa70014, // 0x18: sw $a3,20($sp) + 0xafb00018, // 0x1c: sw $s0,24($sp) + 0xafb1001c, // 0x20: sw $s1,28($sp) + 0xafb20020, // 0x24: sw $s2,32($sp) + 0xafb30024, // 0x28: sw $s3,36($sp) + 0xafb40028, // 0x2c: sw $s4,40($sp) + 0xafb5002c, // 0x30: sw $s5,44($sp) + 0xafb60030, // 0x34: sw $s6,48($sp) + 0xafb70034, // 0x38: sw $s7,52($sp) + 0xafa80038, // 0x3c: sw $t0,56($sp) + 0xafa9003c, // 0x40: sw $t1,60($sp) + 0xafaa0040, // 0x44: sw $t2,64($sp) + 0xafab0044, // 0x48: sw $t3,68($sp) + 0xafac0048, // 0x4c: sw $t4,72($sp) + 0xafad004c, // 0x50: sw $t5,76($sp) + 0xafae0050, // 0x54: sw $t6,80($sp) + 0xafaf0054, // 0x58: sw $t7,84($sp) + 0xafb80058, // 0x5c: sw $t8,88($sp) + 0xafb9005c, // 0x60: sw $t9,92($sp) + 0xafbe0060, // 0x64: sw $fp,96($sp) + 0xafbf0064, // 0x68: sw $ra,100($sp) + + // Callback manager addr. + 0x00000000, // 0x6c: lui $a0,callbackmgr + 0x00000000, // 0x70: addiu $a0,$a0,callbackmgr + + 0x03e02825, // 0x74: move $a1, $ra + 0x24a5ffec, // 0x78: addiu $a1,$a1,-20 + + // JIT re-entry fn addr: + 0x00000000, // 0x7c: lui $t9,reentry + 0x00000000, // 0x80: addiu $t9,$t9,reentry + + 0x0320f809, // 0x84: jalr $t9 + 0x00000000, // 0x88: nop + 0x8fbf0064, // 0x8c: lw $ra,100($sp) + 0x8fbe0060, // 0x90: lw $fp,96($sp) + 0x8fb9005c, // 0x94: lw $t9,92($sp) + 0x8fb80058, // 0x98: lw $t8,88($sp) + 0x8faf0054, // 0x9c: lw $t7,84($sp) + 0x8fae0050, // 0xa0: lw $t6,80($sp) + 0x8fad004c, // 0xa4: lw $t5,76($sp) + 0x8fac0048, // 0xa8: lw $t4,72($sp) + 0x8fab0044, // 0xac: lw $t3,68($sp) + 0x8faa0040, // 0xb0: lw $t2,64($sp) + 0x8fa9003c, // 0xb4: lw $t1,60($sp) + 0x8fa80038, // 0xb8: lw $t0,56($sp) + 0x8fb70034, // 0xbc: lw $s7,52($sp) + 0x8fb60030, // 0xc0: lw $s6,48($sp) + 0x8fb5002c, // 0xc4: lw $s5,44($sp) + 0x8fb40028, // 0xc8: lw $s4,40($sp) + 0x8fb30024, // 0xcc: lw $s3,36($sp) + 0x8fb20020, // 0xd0: lw $s2,32($sp) + 0x8fb1001c, // 0xd4: lw $s1,28($sp) + 0x8fb00018, // 0xd8: lw $s0,24($sp) + 0x8fa70014, // 0xdc: lw $a3,20($sp) + 0x8fa60010, // 0xe0: lw $a2,16($sp) + 0x8fa5000c, // 0xe4: lw $a1,12($sp) + 0x8fa40008, // 0xe8: lw $a0,8($sp) + 0x27bd0068, // 0xec: addiu $sp,$sp,104 + 0x0300f825, // 0xf0: move $ra, $t8 + 0x03200008, // 0xf4: jr $t9 + 0x00000000, // 0xf8: move $t9, $v0/v1 + }; + + const unsigned ReentryFnAddrOffset = 0x7c; // JIT re-entry fn addr lui + const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui + const unsigned Offsett = 0xf8; + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + + // Depending on endian return value will be in v0 or v1. + uint32_t MoveVxT9 = isBigEndian ? 0x0060c825 : 0x0040c825; + memcpy(ResolverMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); + + uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); + uint32_t CallMgrLUi = 0x3c040000 | (((CallMgrAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t CallMgrADDiu = 0x24840000 | ((CallMgrAddr) & 0xFFFF); + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); + memcpy(ResolverMem + CallbackMgrAddrOffset + 4, &CallMgrADDiu, + sizeof(CallMgrADDiu)); + + uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); + uint32_t ReentryLUi = 0x3c190000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryADDiu = 0x27390000 | ((ReentryAddr) & 0xFFFF); + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); + memcpy(ResolverMem + ReentryFnAddrOffset + 4, &ReentryADDiu, + sizeof(ReentryADDiu)); +} + +void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, + void *ResolverAddr, + unsigned NumTrampolines) { + + uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); + uint32_t RHiAddr = ((ResolveAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + Trampolines[5 * I + 0] = 0x03e0c025; // move $t8,$ra + Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); // lui $t9,resolveAddr + Trampolines[5 * I + 2] = 0x27390000 | (ResolveAddr & 0xFFFF); // addiu $t9,$t9,resolveAddr + Trampolines[5 * I + 3] = 0x0320f809; // jalr $t9 + Trampolines[5 * I + 4] = 0x00000000; // nop + } +} + +Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9, ptr1 + // lw $t9, %lo(ptr1)($t9) + // jr $t9 + // stub2: + // lui $t9, ptr2 + // lw $t9,%lo(ptr1)($t9) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .word 0x0 + // ptr2: + // .word 0x0 + // + // ... + + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); + uint64_t PtrAddr = reinterpret_cast<uint64_t>(Stub) + NumPages * PageSize; + + for (unsigned I = 0; I < NumStubs; ++I) { + uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[4 * I + 0] = 0x3c190000 | (HiAddr & 0xFFFF); // lui $t9,ptr1 + Stub[4 * I + 1] = 0x8f390000 | (PtrAddr & 0xFFFF); // lw $t9,%lo(ptr1)($t9) + Stub[4 * I + 2] = 0x03200008; // jr $t9 + Stub[4 * I + 3] = 0x00000000; // nop + PtrAddr += 4; + } + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} + +void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, + void *CallbackMgr) { + + const uint32_t ResolverCode[] = { + //resolver_entry: + 0x67bdff30, // 0x00: daddiu $sp,$sp,-208 + 0xffa20000, // 0x04: sd v0,0(sp) + 0xffa30008, // 0x08: sd v1,8(sp) + 0xffa40010, // 0x0c: sd a0,16(sp) + 0xffa50018, // 0x10: sd a1,24(sp) + 0xffa60020, // 0x14: sd a2,32(sp) + 0xffa70028, // 0x18: sd a3,40(sp) + 0xffa80030, // 0x1c: sd a4,48(sp) + 0xffa90038, // 0x20: sd a5,56(sp) + 0xffaa0040, // 0x24: sd a6,64(sp) + 0xffab0048, // 0x28: sd a7,72(sp) + 0xffac0050, // 0x2c: sd t0,80(sp) + 0xffad0058, // 0x30: sd t1,88(sp) + 0xffae0060, // 0x34: sd t2,96(sp) + 0xffaf0068, // 0x38: sd t3,104(sp) + 0xffb00070, // 0x3c: sd s0,112(sp) + 0xffb10078, // 0x40: sd s1,120(sp) + 0xffb20080, // 0x44: sd s2,128(sp) + 0xffb30088, // 0x48: sd s3,136(sp) + 0xffb40090, // 0x4c: sd s4,144(sp) + 0xffb50098, // 0x50: sd s5,152(sp) + 0xffb600a0, // 0x54: sd s6,160(sp) + 0xffb700a8, // 0x58: sd s7,168(sp) + 0xffb800b0, // 0x5c: sd t8,176(sp) + 0xffb900b8, // 0x60: sd t9,184(sp) + 0xffbe00c0, // 0x64: sd fp,192(sp) + 0xffbf00c8, // 0x68: sd ra,200(sp) + + // Callback manager addr. + 0x00000000, // 0x6c: lui $a0,heighest(callbackmgr) + 0x00000000, // 0x70: daddiu $a0,$a0,heigher(callbackmgr) + 0x00000000, // 0x74: dsll $a0,$a0,16 + 0x00000000, // 0x78: daddiu $a0,$a0,hi(callbackmgr) + 0x00000000, // 0x7c: dsll $a0,$a0,16 + 0x00000000, // 0x80: daddiu $a0,$a0,lo(callbackmgr) + + 0x03e02825, // 0x84: move $a1, $ra + 0x64a5ffdc, // 0x88: daddiu $a1,$a1,-36 + + // JIT re-entry fn addr: + 0x00000000, // 0x8c: lui $t9,reentry + 0x00000000, // 0x90: daddiu $t9,$t9,reentry + 0x00000000, // 0x94: dsll $t9,$t9, + 0x00000000, // 0x98: daddiu $t9,$t9, + 0x00000000, // 0x9c: dsll $t9,$t9, + 0x00000000, // 0xa0: daddiu $t9,$t9, + 0x0320f809, // 0xa4: jalr $t9 + 0x00000000, // 0xa8: nop + 0xdfbf00c8, // 0xac: ld ra, 200(sp) + 0xdfbe00c0, // 0xb0: ld fp, 192(sp) + 0xdfb900b8, // 0xb4: ld t9, 184(sp) + 0xdfb800b0, // 0xb8: ld t8, 176(sp) + 0xdfb700a8, // 0xbc: ld s7, 168(sp) + 0xdfb600a0, // 0xc0: ld s6, 160(sp) + 0xdfb50098, // 0xc4: ld s5, 152(sp) + 0xdfb40090, // 0xc8: ld s4, 144(sp) + 0xdfb30088, // 0xcc: ld s3, 136(sp) + 0xdfb20080, // 0xd0: ld s2, 128(sp) + 0xdfb10078, // 0xd4: ld s1, 120(sp) + 0xdfb00070, // 0xd8: ld s0, 112(sp) + 0xdfaf0068, // 0xdc: ld t3, 104(sp) + 0xdfae0060, // 0xe0: ld t2, 96(sp) + 0xdfad0058, // 0xe4: ld t1, 88(sp) + 0xdfac0050, // 0xe8: ld t0, 80(sp) + 0xdfab0048, // 0xec: ld a7, 72(sp) + 0xdfaa0040, // 0xf0: ld a6, 64(sp) + 0xdfa90038, // 0xf4: ld a5, 56(sp) + 0xdfa80030, // 0xf8: ld a4, 48(sp) + 0xdfa70028, // 0xfc: ld a3, 40(sp) + 0xdfa60020, // 0x100: ld a2, 32(sp) + 0xdfa50018, // 0x104: ld a1, 24(sp) + 0xdfa40010, // 0x108: ld a0, 16(sp) + 0xdfa30008, // 0x10c: ld v1, 8(sp) + 0x67bd00d0, // 0x110: daddiu $sp,$sp,208 + 0x0300f825, // 0x114: move $ra, $t8 + 0x03200008, // 0x118: jr $t9 + 0x0040c825, // 0x11c: move $t9, $v0 + }; + + const unsigned ReentryFnAddrOffset = 0x8c; // JIT re-entry fn addr lui + const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui + + memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + + uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); + + uint32_t CallMgrLUi = + 0x3c040000 | (((CallMgrAddr + 0x800080008000) >> 48) & 0xFFFF); + uint32_t CallMgrDADDiu = + 0x64840000 | (((CallMgrAddr + 0x80008000) >> 32) & 0xFFFF); + uint32_t CallMgrDSLL = 0x00042438; + uint32_t CallMgrDADDiu2 = + 0x64840000 | ((((CallMgrAddr + 0x8000) >> 16) & 0xFFFF)); + uint32_t CallMgrDSLL2 = 0x00042438; + uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr)&0xFFFF); + + memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrDADDiu, + sizeof(CallMgrDADDiu)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 8), &CallMgrDSLL, + sizeof(CallMgrDSLL)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 12), &CallMgrDADDiu2, + sizeof(CallMgrDADDiu2)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 16), &CallMgrDSLL2, + sizeof(CallMgrDSLL2)); + memcpy(ResolverMem + (CallbackMgrAddrOffset + 20), &CallMgrDADDiu3, + sizeof(CallMgrDADDiu3)); + + uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); + + uint32_t ReentryLUi = + 0x3c190000 | (((ReentryAddr + 0x800080008000) >> 48) & 0xFFFF); + + uint32_t ReentryDADDiu = + 0x67390000 | (((ReentryAddr + 0x80008000) >> 32) & 0xFFFF); + + uint32_t ReentryDSLL = 0x0019cc38; + + uint32_t ReentryDADDiu2 = + 0x67390000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); + + uint32_t ReentryDSLL2 = 0x0019cc38; + + uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr)&0xFFFF); + + memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryDADDiu, + sizeof(ReentryDADDiu)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 8), &ReentryDSLL, + sizeof(ReentryDSLL)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 12), &ReentryDADDiu2, + sizeof(ReentryDADDiu2)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 16), &ReentryDSLL2, + sizeof(ReentryDSLL2)); + memcpy(ResolverMem + (ReentryFnAddrOffset + 20), &ReentryDADDiu3, + sizeof(ReentryDADDiu3)); +} + +void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines) { + + uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); + + uint64_t HeighestAddr = ((ResolveAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolveAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolveAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra + Trampolines[10 * I + 1] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,resolveAddr + Trampolines[10 * I + 2] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(resolveAddr) + Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 6] = 0x67390000 | (ResolveAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 + Trampolines[10 * I + 8] = 0x00000000; // nop + Trampolines[10 * I + 9] = 0x00000000; // nop + } +} + +Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // stub2: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .dword 0x0 + // ptr2: + // .dword 0x0 + // + // ... + const unsigned StubSize = IndirectStubsInfo::StubSize; + + // Emit at least MinStubs, rounded up to fill the pages allocated. + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); + unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; + unsigned NumStubs = (NumPages * PageSize) / StubSize; + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + 2 * NumPages * PageSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + + if (EC) + return errorCodeToError(EC); + + // Create separate MemoryBlocks representing the stubs and pointers. + sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); + sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + + NumPages * PageSize, + NumPages * PageSize); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); + uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); + + for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { + uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((PtrAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[8 * I + 0] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,ptr1 + Stub[8 * I + 1] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(ptr) + Stub[8 * I + 2] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 3] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Stub[8 * I + 4] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 5] = 0xdf390000 | (PtrAddr & 0xFFFF); // ld $t9,%lo(ptr) + Stub[8 * I + 6] = 0x03200008; // jr $t9 + Stub[8 * I + 7] = 0x00000000; // nop + } + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + // Initialize all pointers to point at FailureAddress. + void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); + for (unsigned I = 0; I < NumStubs; ++I) + Ptr[I] = InitialPtrVal; + + StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); + + return Error::success(); +} +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp new file mode 100644 index 000000000000..28c8479abba4 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -0,0 +1,158 @@ +//===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcCBindingsStack.h" +#include "llvm-c/OrcBindings.h" +#include "llvm/ExecutionEngine/JITEventListener.h" + +using namespace llvm; + +LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { + TargetMachine *TM2(unwrap(TM)); + + Triple T(TM2->getTargetTriple()); + + auto IndirectStubsMgrBuilder = + orc::createLocalIndirectStubsManagerBuilder(T); + + OrcCBindingsStack *JITStack = + new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder)); + + return wrap(JITStack); +} + +const char *LLVMOrcGetErrorMsg(LLVMOrcJITStackRef JITStack) { + OrcCBindingsStack &J = *unwrap(JITStack); + return J.getErrorMessage().c_str(); +} + +void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledName, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + std::string Mangled = J.mangle(SymbolName); + *MangledName = new char[Mangled.size() + 1]; + strcpy(*MangledName, Mangled.c_str()); +} + +void LLVMOrcDisposeMangledSymbol(char *MangledName) { delete[] MangledName; } + +LLVMErrorRef LLVMOrcCreateLazyCompileCallback( + LLVMOrcJITStackRef JITStack, LLVMOrcTargetAddress *RetAddr, + LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + if (auto Addr = J.createLazyCompileCallback(Callback, CallbackCtx)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); +} + +LLVMErrorRef LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress InitAddr) { + OrcCBindingsStack &J = *unwrap(JITStack); + return wrap(J.createIndirectStub(StubName, InitAddr)); +} + +LLVMErrorRef LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress NewAddr) { + OrcCBindingsStack &J = *unwrap(JITStack); + return wrap(J.setIndirectStubPointer(StubName, NewAddr)); +} + +LLVMErrorRef LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + std::unique_ptr<Module> M(unwrap(Mod)); + if (auto Handle = + J.addIRModuleEager(std::move(M), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); +} + +LLVMErrorRef LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + std::unique_ptr<Module> M(unwrap(Mod)); + if (auto Handle = + J.addIRModuleLazy(std::move(M), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); +} + +LLVMErrorRef LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle *RetHandle, + LLVMMemoryBufferRef Obj, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx) { + OrcCBindingsStack &J = *unwrap(JITStack); + std::unique_ptr<MemoryBuffer> O(unwrap(Obj)); + if (auto Handle = + J.addObject(std::move(O), SymbolResolver, SymbolResolverCtx)) { + *RetHandle = *Handle; + return LLVMErrorSuccess; + } else + return wrap(Handle.takeError()); +} + +LLVMErrorRef LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, + LLVMOrcModuleHandle H) { + OrcCBindingsStack &J = *unwrap(JITStack); + return wrap(J.removeModule(H)); +} + +LLVMErrorRef LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + if (auto Addr = J.findSymbolAddress(SymbolName, true)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); +} + +LLVMErrorRef LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + LLVMOrcModuleHandle H, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + if (auto Addr = J.findSymbolAddressIn(H, SymbolName, true)) { + *RetAddr = *Addr; + return LLVMErrorSuccess; + } else + return wrap(Addr.takeError()); +} + +LLVMErrorRef LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { + auto *J = unwrap(JITStack); + auto Err = J->shutdown(); + delete J; + return wrap(std::move(Err)); +} + +void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->RegisterJITEventListener(unwrap(L)); +} + +void LLVMOrcUnregisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->UnregisterJITEventListener(unwrap(L)); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h new file mode 100644 index 000000000000..98129e1690d2 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -0,0 +1,533 @@ +//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H +#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H + +#include "llvm-c/OrcBindings.h" +#include "llvm-c/TargetMachine.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include <algorithm> +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace llvm { + +class OrcCBindingsStack; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +namespace detail { + +// FIXME: Kill this off once the Layer concept becomes an interface. +class GenericLayer { +public: + virtual ~GenericLayer() = default; + + virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) = 0; + virtual Error removeModule(orc::VModuleKey K) = 0; + }; + + template <typename LayerT> class GenericLayerImpl : public GenericLayer { + public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) override { + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); + } + + Error removeModule(orc::VModuleKey K) override { + return Layer.removeModule(K); + } + + private: + LayerT &Layer; + }; + + template <> + class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer { + private: + using LayerT = orc::LegacyRTDyldObjectLinkingLayer; + public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) override { + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); + } + + Error removeModule(orc::VModuleKey K) override { + return Layer.removeObject(K); + } + + private: + LayerT &Layer; + }; + + template <typename LayerT> + std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { + return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer); + } + +} // end namespace detail + +class OrcCBindingsStack { +public: + + using CompileCallbackMgr = orc::JITCompileCallbackManager; + using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer; + using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>; + using CODLayerT = + orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>; + + using CallbackManagerBuilder = + std::function<std::unique_ptr<CompileCallbackMgr>()>; + + using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT; + +private: + + using OwningObject = object::OwningBinary<object::ObjectFile>; + + class CBindingsResolver : public orc::SymbolResolver { + public: + CBindingsResolver(OrcCBindingsStack &Stack, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) + : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), + ExternalResolverCtx(std::move(ExternalResolverCtx)) {} + + orc::SymbolNameSet + getResponsibilitySet(const orc::SymbolNameSet &Symbols) override { + orc::SymbolNameSet Result; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (!Sym.getFlags().isStrong()) + Result.insert(S); + } else if (auto Err = Sym.takeError()) { + Stack.reportError(std::move(Err)); + return orc::SymbolNameSet(); + } + } + + return Result; + } + + orc::SymbolNameSet + lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query, + orc::SymbolNameSet Symbols) override { + orc::SymbolNameSet UnresolvedSymbols; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + } else { + Stack.ES.legacyFailQuery(*Query, Addr.takeError()); + return orc::SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + Stack.ES.legacyFailQuery(*Query, std::move(Err)); + return orc::SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + + if (Query->isComplete()) + Query->handleComplete(); + + return UnresolvedSymbols; + } + + private: + JITSymbol findSymbol(const std::string &Name) { + // Search order: + // 1. JIT'd symbols. + // 2. Runtime overrides. + // 3. External resolver (if present). + + if (Stack.CODLayer) { + if (auto Sym = Stack.CODLayer->findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } else { + if (auto Sym = Stack.CompileLayer.findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + } + + if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) + return Sym; + + if (ExternalResolver) + return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx), + JITSymbolFlags::Exported); + + return JITSymbol(nullptr); + } + + OrcCBindingsStack &Stack; + LLVMOrcSymbolResolverFn ExternalResolver; + void *ExternalResolverCtx = nullptr; + }; + +public: + OrcCBindingsStack(TargetMachine &TM, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder) + : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), + IndirectStubsMgr(IndirectStubsMgrBuilder()), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + orc::SimpleCompiler(TM)), + CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), + std::move(IndirectStubsMgrBuilder), Resolvers)), + CXXRuntimeOverrides( + AcknowledgeORCv1Deprecation, + [this](const std::string &S) { return mangle(S); }) {} + + Error shutdown() { + // Run any destructors registered with __cxa_atexit. + CXXRuntimeOverrides.runDestructors(); + // Run any IR destructors. + for (auto &DtorRunner : IRStaticDestructorRunners) + if (auto Err = DtorRunner.runViaLayer(*this)) + return Err; + return Error::success(); + } + + std::string mangle(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return MangledName; + } + + template <typename PtrTy> + static PtrTy fromTargetAddress(JITTargetAddress Addr) { + return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr)); + } + + Expected<JITTargetAddress> + createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, + void *CallbackCtx) { + auto WrappedCallback = [=]() -> JITTargetAddress { + return Callback(wrap(this), CallbackCtx); + }; + + return CCMgr->getCompileCallback(std::move(WrappedCallback)); + } + + Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) { + return IndirectStubsMgr->createStub(StubName, Addr, + JITSymbolFlags::Exported); + } + + Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) { + return IndirectStubsMgr->updatePointer(Name, Addr); + } + + template <typename LayerT> + Expected<orc::VModuleKey> + addIRModule(LayerT &Layer, std::unique_ptr<Module> M, + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + + // Attach a data-layout if one isn't already present. + if (M->getDataLayout().isDefault()) + M->setDataLayout(DL); + + // Record the static constructors and destructors. We have to do this before + // we hand over ownership of the module to the JIT. + std::vector<std::string> CtorNames, DtorNames; + for (auto Ctor : orc::getConstructors(*M)) + CtorNames.push_back(mangle(Ctor.Func->getName())); + for (auto Dtor : orc::getDestructors(*M)) + DtorNames.push_back(mangle(Dtor.Func->getName())); + + // Add the module to the JIT. + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver, + ExternalResolverCtx); + if (auto Err = Layer.addModule(K, std::move(M))) + return std::move(Err); + + KeyLayers[K] = detail::createGenericLayer(Layer); + + // Run the static constructors, and save the static destructor runner for + // execution when the JIT is torn down. + orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner( + AcknowledgeORCv1Deprecation, std::move(CtorNames), K); + if (auto Err = CtorRunner.runViaLayer(*this)) + return std::move(Err); + + IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K); + + return K; + } + + Expected<orc::VModuleKey> + addIRModuleEager(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + return addIRModule(CompileLayer, std::move(M), + llvm::make_unique<SectionMemoryManager>(), + std::move(ExternalResolver), ExternalResolverCtx); + } + + Expected<orc::VModuleKey> + addIRModuleLazy(std::unique_ptr<Module> M, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + if (!CODLayer) + return make_error<StringError>("Can not add lazy module: No compile " + "callback manager available", + inconvertibleErrorCode()); + + return addIRModule(*CODLayer, std::move(M), + llvm::make_unique<SectionMemoryManager>(), + std::move(ExternalResolver), ExternalResolverCtx); + } + + Error removeModule(orc::VModuleKey K) { + // FIXME: Should error release the module key? + if (auto Err = KeyLayers[K]->removeModule(K)) + return Err; + ES.releaseVModule(K); + KeyLayers.erase(K); + return Error::success(); + } + + Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) { + if (auto Obj = object::ObjectFile::createObjectFile( + ObjBuffer->getMemBufferRef())) { + + auto K = ES.allocateVModule(); + Resolvers[K] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); + + if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer))) + return std::move(Err); + + KeyLayers[K] = detail::createGenericLayer(ObjectLayer); + + return K; + } else + return Obj.takeError(); + } + + JITSymbol findSymbol(const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) + return Sym; + if (CODLayer) + return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly); + return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); + } + + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) { + assert(KeyLayers.count(K) && "looking up symbol in unknown module"); + return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); + } + + Expected<JITTargetAddress> findSymbolAddress(const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return std::move(Err); + } + + // No symbol not found. Return 0. + return 0; + } + + Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K, + const std::string &Name, + bool ExportedSymbolsOnly) { + if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) + return *AddrOrErr; + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return std::move(Err); + } + + // Symbol not found. Return 0. + return 0; + } + + const std::string &getErrorMessage() const { return ErrMsg; } + + void RegisterJITEventListener(JITEventListener *L) { + if (!L) + return; + EventListeners.push_back(L); + } + + void UnregisterJITEventListener(JITEventListener *L) { + if (!L) + return; + + auto I = find(reverse(EventListeners), L); + if (I != EventListeners.rend()) { + std::swap(*I, EventListeners.back()); + EventListeners.pop_back(); + } + } + +private: + using ResolverMap = + std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>; + + static std::unique_ptr<CompileCallbackMgr> + createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) { + auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0); + if (!CCMgr) { + // FIXME: It would be good if we could report this somewhere, but we do + // have an instance yet. + logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: "); + return nullptr; + } + return std::move(*CCMgr); + } + + static std::unique_ptr<CODLayerT> + createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer, + CompileCallbackMgr *CCMgr, + IndirectStubsManagerBuilder IndirectStubsMgrBuilder, + ResolverMap &Resolvers) { + // If there is no compile callback manager available we can not create a + // compile on demand layer. + if (!CCMgr) + return nullptr; + + return llvm::make_unique<CODLayerT>( + AcknowledgeORCv1Deprecation, ES, CompileLayer, + [&Resolvers](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && "No resolver for module K"); + return ResolverI->second; + }, + [&Resolvers](orc::VModuleKey K, + std::shared_ptr<orc::SymbolResolver> Resolver) { + assert(!Resolvers.count(K) && "Resolver already present"); + Resolvers[K] = std::move(Resolver); + }, + [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr, + std::move(IndirectStubsMgrBuilder), false); + } + + void reportError(Error Err) { + // FIXME: Report errors on the execution session. + logAllUnhandledErrors(std::move(Err), errs(), "ORC error: "); + }; + + void notifyFinalized(orc::VModuleKey K, + const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); + for (auto &Listener : EventListeners) + Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo); + } + + void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { + uint64_t Key = static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(Obj.getData().data())); + for (auto &Listener : EventListeners) + Listener->notifyFreeingObject(Key); + } + + orc::ExecutionSession ES; + std::unique_ptr<CompileCallbackMgr> CCMgr; + + std::vector<JITEventListener *> EventListeners; + + DataLayout DL; + SectionMemoryManager CCMgrMemMgr; + + std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; + + ObjLayerT ObjectLayer; + CompileLayerT CompileLayer; + std::unique_ptr<CODLayerT> CODLayer; + + std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; + + orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides; + std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; + std::string ErrMsg; + + ResolverMap Resolvers; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcError.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcError.cpp new file mode 100644 index 000000000000..e6e9a095319c --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcError.cpp @@ -0,0 +1,115 @@ +//===---------------- OrcError.cpp - Error codes for ORC ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Error codes for ORC. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class OrcErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "orc"; } + + std::string message(int condition) const override { + switch (static_cast<OrcErrorCode>(condition)) { + case OrcErrorCode::UnknownORCError: + return "Unknown ORC error"; + case OrcErrorCode::DuplicateDefinition: + return "Duplicate symbol definition"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; + case OrcErrorCode::RemoteAllocatorDoesNotExist: + return "Remote allocator does not exist"; + case OrcErrorCode::RemoteAllocatorIdAlreadyInUse: + return "Remote allocator Id already in use"; + case OrcErrorCode::RemoteMProtectAddrUnrecognized: + return "Remote mprotect call references unallocated memory"; + case OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist: + return "Remote indirect stubs owner does not exist"; + case OrcErrorCode::RemoteIndirectStubsOwnerIdAlreadyInUse: + return "Remote indirect stubs owner Id already in use"; + case OrcErrorCode::RPCConnectionClosed: + return "RPC connection closed"; + case OrcErrorCode::RPCCouldNotNegotiateFunction: + return "Could not negotiate RPC function"; + case OrcErrorCode::RPCResponseAbandoned: + return "RPC response abandoned"; + case OrcErrorCode::UnexpectedRPCCall: + return "Unexpected RPC call"; + case OrcErrorCode::UnexpectedRPCResponse: + return "Unexpected RPC response"; + case OrcErrorCode::UnknownErrorCodeFromRemote: + return "Unknown error returned from remote RPC function " + "(Use StringError to get error message)"; + case OrcErrorCode::UnknownResourceHandle: + return "Unknown resource handle"; + } + llvm_unreachable("Unhandled error code"); + } +}; + +static ManagedStatic<OrcErrorCategory> OrcErrCat; +} + +namespace llvm { +namespace orc { + +char DuplicateDefinition::ID = 0; +char JITSymbolNotFound::ID = 0; + +std::error_code orcError(OrcErrorCode ErrCode) { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat); +} + + +DuplicateDefinition::DuplicateDefinition(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code DuplicateDefinition::convertToErrorCode() const { + return orcError(OrcErrorCode::DuplicateDefinition); +} + +void DuplicateDefinition::log(raw_ostream &OS) const { + OS << "Duplicate definition of symbol '" << SymbolName << "'"; +} + +const std::string &DuplicateDefinition::getSymbolName() const { + return SymbolName; +} + +JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code JITSymbolNotFound::convertToErrorCode() const { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(OrcErrorCode::JITSymbolNotFound), + *OrcErrCat); +} + +void JITSymbolNotFound::log(raw_ostream &OS) const { + OS << "Could not find symbol '" << SymbolName << "'"; +} + +const std::string &JITSymbolNotFound::getSymbolName() const { + return SymbolName; +} + +} +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp new file mode 100644 index 000000000000..772a9c2c4ab2 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -0,0 +1,138 @@ +//===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcMCJITReplacement.h" +#include "llvm/ExecutionEngine/GenericValue.h" + +namespace { + +static struct RegisterJIT { + RegisterJIT() { llvm::orc::OrcMCJITReplacement::Register(); } +} JITRegistrator; + +} + +extern "C" void LLVMLinkInOrcMCJITReplacement() {} + +namespace llvm { +namespace orc { + +GenericValue +OrcMCJITReplacement::runFunction(Function *F, + ArrayRef<GenericValue> ArgValues) { + assert(F && "Function *F was null at entry to run()"); + + void *FPtr = getPointerToFunction(F); + assert(FPtr && "Pointer to fn's code was null after getPointerToFunction"); + FunctionType *FTy = F->getFunctionType(); + Type *RetTy = FTy->getReturnType(); + + assert((FTy->getNumParams() == ArgValues.size() || + (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) && + "Wrong number of arguments passed into function!"); + assert(FTy->getNumParams() == ArgValues.size() && + "This doesn't support passing arguments through varargs (yet)!"); + + // Handle some common cases first. These cases correspond to common `main' + // prototypes. + if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) { + switch (ArgValues.size()) { + case 3: + if (FTy->getParamType(0)->isIntegerTy(32) && + FTy->getParamType(1)->isPointerTy() && + FTy->getParamType(2)->isPointerTy()) { + int (*PF)(int, char **, const char **) = + (int (*)(int, char **, const char **))(intptr_t)FPtr; + + // Call the function. + GenericValue rv; + rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), + (char **)GVTOP(ArgValues[1]), + (const char **)GVTOP(ArgValues[2]))); + return rv; + } + break; + case 2: + if (FTy->getParamType(0)->isIntegerTy(32) && + FTy->getParamType(1)->isPointerTy()) { + int (*PF)(int, char **) = (int (*)(int, char **))(intptr_t)FPtr; + + // Call the function. + GenericValue rv; + rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), + (char **)GVTOP(ArgValues[1]))); + return rv; + } + break; + case 1: + if (FTy->getNumParams() == 1 && FTy->getParamType(0)->isIntegerTy(32)) { + GenericValue rv; + int (*PF)(int) = (int (*)(int))(intptr_t)FPtr; + rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue())); + return rv; + } + break; + } + } + + // Handle cases where no arguments are passed first. + if (ArgValues.empty()) { + GenericValue rv; + switch (RetTy->getTypeID()) { + default: + llvm_unreachable("Unknown return type for function call!"); + case Type::IntegerTyID: { + unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth(); + if (BitWidth == 1) + rv.IntVal = APInt(BitWidth, ((bool (*)())(intptr_t)FPtr)()); + else if (BitWidth <= 8) + rv.IntVal = APInt(BitWidth, ((char (*)())(intptr_t)FPtr)()); + else if (BitWidth <= 16) + rv.IntVal = APInt(BitWidth, ((short (*)())(intptr_t)FPtr)()); + else if (BitWidth <= 32) + rv.IntVal = APInt(BitWidth, ((int (*)())(intptr_t)FPtr)()); + else if (BitWidth <= 64) + rv.IntVal = APInt(BitWidth, ((int64_t (*)())(intptr_t)FPtr)()); + else + llvm_unreachable("Integer types > 64 bits not supported"); + return rv; + } + case Type::VoidTyID: + rv.IntVal = APInt(32, ((int (*)())(intptr_t)FPtr)()); + return rv; + case Type::FloatTyID: + rv.FloatVal = ((float (*)())(intptr_t)FPtr)(); + return rv; + case Type::DoubleTyID: + rv.DoubleVal = ((double (*)())(intptr_t)FPtr)(); + return rv; + case Type::X86_FP80TyID: + case Type::FP128TyID: + case Type::PPC_FP128TyID: + llvm_unreachable("long double not supported yet"); + case Type::PointerTyID: + return PTOGV(((void *(*)())(intptr_t)FPtr)()); + } + } + + llvm_unreachable("Full-featured argument passing not supported yet!"); +} + +void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { + auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; + + for (auto &KV : CtorDtorsMap) + cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>( + AcknowledgeORCv1Deprecation, std::move(KV.second), KV.first) + .runViaLayer(LazyEmitLayer)); + + CtorDtorsMap.clear(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h new file mode 100644 index 000000000000..169dc8f1d02b --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -0,0 +1,501 @@ +//===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Orc based MCJIT replacement. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H +#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/GenericValue.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Archive.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace llvm { + +class ObjectCache; + +namespace orc { + +class OrcMCJITReplacement : public ExecutionEngine { + + // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that + // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are + // expecting - see finalizeMemory. + class MCJITReplacementMemMgr : public MCJITMemoryManager { + public: + MCJITReplacementMemMgr(OrcMCJITReplacement &M, + std::shared_ptr<MCJITMemoryManager> ClientMM) + : M(M), ClientMM(std::move(ClientMM)) {} + + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override { + uint8_t *Addr = + ClientMM->allocateCodeSection(Size, Alignment, SectionID, + SectionName); + M.SectionsAllocatedSinceLastLoad.insert(Addr); + return Addr; + } + + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool IsReadOnly) override { + uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID, + SectionName, IsReadOnly); + M.SectionsAllocatedSinceLastLoad.insert(Addr); + return Addr; + } + + void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override { + return ClientMM->reserveAllocationSpace(CodeSize, CodeAlign, + RODataSize, RODataAlign, + RWDataSize, RWDataAlign); + } + + bool needsToReserveAllocationSpace() override { + return ClientMM->needsToReserveAllocationSpace(); + } + + void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, + size_t Size) override { + return ClientMM->registerEHFrames(Addr, LoadAddr, Size); + } + + void deregisterEHFrames() override { + return ClientMM->deregisterEHFrames(); + } + + void notifyObjectLoaded(RuntimeDyld &RTDyld, + const object::ObjectFile &O) override { + return ClientMM->notifyObjectLoaded(RTDyld, O); + } + + void notifyObjectLoaded(ExecutionEngine *EE, + const object::ObjectFile &O) override { + return ClientMM->notifyObjectLoaded(EE, O); + } + + bool finalizeMemory(std::string *ErrMsg = nullptr) override { + // Each set of objects loaded will be finalized exactly once, but since + // symbol lookup during relocation may recursively trigger the + // loading/relocation of other modules, and since we're forwarding all + // finalizeMemory calls to a single underlying memory manager, we need to + // defer forwarding the call on until all necessary objects have been + // loaded. Otherwise, during the relocation of a leaf object, we will end + // up finalizing memory, causing a crash further up the stack when we + // attempt to apply relocations to finalized memory. + // To avoid finalizing too early, look at how many objects have been + // loaded but not yet finalized. This is a bit of a hack that relies on + // the fact that we're lazily emitting object files: The only way you can + // get more than one set of objects loaded but not yet finalized is if + // they were loaded during relocation of another set. + if (M.UnfinalizedSections.size() == 1) + return ClientMM->finalizeMemory(ErrMsg); + return false; + } + + private: + OrcMCJITReplacement &M; + std::shared_ptr<MCJITMemoryManager> ClientMM; + }; + + class LinkingORCResolver : public orc::SymbolResolver { + public: + LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} + + SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override { + SymbolNameSet Result; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + if (!Sym.getFlags().isStrong()) + Result.insert(S); + } else if (auto Err = Sym.takeError()) { + M.reportError(std::move(Err)); + return SymbolNameSet(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) { + if (!Sym2.getFlags().isStrong()) + Result.insert(S); + } else if (auto Err = Sym2.takeError()) { + M.reportError(std::move(Err)); + return SymbolNameSet(); + } else + Result.insert(S); + } + } + + return Result; + } + + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) override { + SymbolNameSet UnresolvedSymbols; + bool NewSymbolsResolved = false; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { + if (auto Addr = Sym2.getAddress()) { + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym2.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + } + + if (NewSymbolsResolved && Query->isComplete()) + Query->handleComplete(); + + return UnresolvedSymbols; + } + + private: + OrcMCJITReplacement &M; + }; + +private: + static ExecutionEngine * + createOrcMCJITReplacement(std::string *ErrorMsg, + std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, + std::unique_ptr<TargetMachine> TM) { + return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), + std::move(TM)); + } + + void reportError(Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: "); + } + +public: + OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, + std::unique_ptr<TargetMachine> TM) + : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), + MemMgr( + std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), + Resolver(std::make_shared<LinkingORCResolver>(*this)), + ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), + NotifyFinalized(*this), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](VModuleKey K) { + return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; + }, + NotifyObjectLoaded, NotifyFinalized), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + SimpleCompiler(*this->TM), + [this](VModuleKey K, std::unique_ptr<Module> M) { + Modules.push_back(std::move(M)); + }), + LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} + + static void Register() { + OrcMCJITReplacementCtor = createOrcMCJITReplacement; + } + + void addModule(std::unique_ptr<Module> M) override { + // If this module doesn't have a DataLayout attached then attach the + // default. + if (M->getDataLayout().isDefault()) { + M->setDataLayout(getDataLayout()); + } else { + assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); + } + + // Rename, bump linkage and record static constructors and destructors. + // We have to do this before we hand over ownership of the module to the + // JIT. + std::vector<std::string> CtorNames, DtorNames; + { + unsigned CtorId = 0, DtorId = 0; + for (auto Ctor : orc::getConstructors(*M)) { + std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str(); + Ctor.Func->setName(NewCtorName); + Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); + Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); + CtorNames.push_back(mangle(NewCtorName)); + } + for (auto Dtor : orc::getDestructors(*M)) { + std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str(); + dbgs() << "Found dtor: " << NewDtorName << "\n"; + Dtor.Func->setName(NewDtorName); + Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); + Dtor.Func->setVisibility(GlobalValue::HiddenVisibility); + DtorNames.push_back(mangle(NewDtorName)); + } + } + + auto K = ES.allocateVModule(); + + UnexecutedConstructors[K] = std::move(CtorNames); + UnexecutedDestructors[K] = std::move(DtorNames); + + cantFail(LazyEmitLayer.addModule(K, std::move(M))); + } + + void addObjectFile(std::unique_ptr<object::ObjectFile> O) override { + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData()))); + } + + void addObjectFile(object::OwningBinary<object::ObjectFile> O) override { + std::unique_ptr<object::ObjectFile> Obj; + std::unique_ptr<MemoryBuffer> ObjBuffer; + std::tie(Obj, ObjBuffer) = O.takeBinary(); + cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer))); + } + + void addArchive(object::OwningBinary<object::Archive> A) override { + Archives.push_back(std::move(A)); + } + + bool removeModule(Module *M) override { + auto I = Modules.begin(); + for (auto E = Modules.end(); I != E; ++I) + if (I->get() == M) + break; + if (I == Modules.end()) + return false; + Modules.erase(I); + return true; + } + + uint64_t getSymbolAddress(StringRef Name) { + return cantFail(findSymbol(Name).getAddress()); + } + + JITSymbol findSymbol(StringRef Name) { + return findMangledSymbol(mangle(Name)); + } + + void finalizeObject() override { + // This is deprecated - Aim to remove in ExecutionEngine. + // REMOVE IF POSSIBLE - Doesn't make sense for New JIT. + } + + void mapSectionAddress(const void *LocalAddress, + uint64_t TargetAddress) override { + for (auto &P : UnfinalizedSections) + if (P.second.count(LocalAddress)) + ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress); + } + + uint64_t getGlobalValueAddress(const std::string &Name) override { + return getSymbolAddress(Name); + } + + uint64_t getFunctionAddress(const std::string &Name) override { + return getSymbolAddress(Name); + } + + void *getPointerToFunction(Function *F) override { + uint64_t FAddr = getSymbolAddress(F->getName()); + return reinterpret_cast<void *>(static_cast<uintptr_t>(FAddr)); + } + + void *getPointerToNamedFunction(StringRef Name, + bool AbortOnFailure = true) override { + uint64_t Addr = getSymbolAddress(Name); + if (!Addr && AbortOnFailure) + llvm_unreachable("Missing symbol!"); + return reinterpret_cast<void *>(static_cast<uintptr_t>(Addr)); + } + + GenericValue runFunction(Function *F, + ArrayRef<GenericValue> ArgValues) override; + + void setObjectCache(ObjectCache *NewCache) override { + CompileLayer.getCompiler().setObjectCache(NewCache); + } + + void setProcessAllSections(bool ProcessAllSections) override { + ObjectLayer.setProcessAllSections(ProcessAllSections); + } + + void runStaticConstructorsDestructors(bool isDtors) override; + +private: + JITSymbol findMangledSymbol(StringRef Name) { + if (auto Sym = LazyEmitLayer.findSymbol(Name, false)) + return Sym; + if (auto Sym = ClientResolver->findSymbol(Name)) + return Sym; + if (auto Sym = scanArchives(Name)) + return Sym; + + return nullptr; + } + + JITSymbol scanArchives(StringRef Name) { + for (object::OwningBinary<object::Archive> &OB : Archives) { + object::Archive *A = OB.getBinary(); + // Look for our symbols in each Archive + auto OptionalChildOrErr = A->findSym(Name); + if (!OptionalChildOrErr) + report_fatal_error(OptionalChildOrErr.takeError()); + auto &OptionalChild = *OptionalChildOrErr; + if (OptionalChild) { + // FIXME: Support nested archives? + Expected<std::unique_ptr<object::Binary>> ChildBinOrErr = + OptionalChild->getAsBinary(); + if (!ChildBinOrErr) { + // TODO: Actually report errors helpfully. + consumeError(ChildBinOrErr.takeError()); + continue; + } + std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); + if (ChildBin->isObject()) { + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), + MemoryBuffer::getMemBufferCopy(ChildBin->getData()))); + if (auto Sym = ObjectLayer.findSymbol(Name, true)) + return Sym; + } + } + } + return nullptr; + } + + class NotifyObjectLoadedT { + public: + using LoadedObjInfoListT = + std::vector<std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>; + + NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {} + + void operator()(VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &Info) const { + M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad); + M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); + M.MemMgr->notifyObjectLoaded(&M, Obj); + } + private: + OrcMCJITReplacement &M; + }; + + class NotifyFinalizedT { + public: + NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {} + + void operator()(VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &Info) { + M.UnfinalizedSections.erase(K); + } + + private: + OrcMCJITReplacement &M; + }; + + std::string mangle(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout()); + } + return MangledName; + } + + using ObjectLayerT = LegacyRTDyldObjectLinkingLayer; + using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; + using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; + + ExecutionSession ES; + + std::unique_ptr<TargetMachine> TM; + std::shared_ptr<MCJITReplacementMemMgr> MemMgr; + std::shared_ptr<LinkingORCResolver> Resolver; + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; + Mangler Mang; + + // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr + // delete blocks in LocalModules refer to the ShouldDelete map, so + // LocalModules needs to be destructed before ShouldDelete. + std::map<Module*, bool> ShouldDelete; + + NotifyObjectLoadedT NotifyObjectLoaded; + NotifyFinalizedT NotifyFinalized; + + ObjectLayerT ObjectLayer; + CompileLayerT CompileLayer; + LazyEmitLayerT LazyEmitLayer; + + std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors; + std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors; + + // We need to store ObjLayerT::ObjSetHandles for each of the object sets + // that have been emitted but not yet finalized so that we can forward the + // mapSectionAddress calls appropriately. + using SectionAddrSet = std::set<const void *>; + SectionAddrSet SectionsAllocatedSinceLastLoad; + std::map<VModuleKey, SectionAddrSet> UnfinalizedSections; + + std::vector<object::OwningBinary<object::Archive>> Archives; +}; + +} // end namespace orc + +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_ORC_MCJITREPLACEMENT_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp new file mode 100644 index 000000000000..367b3639f841 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp @@ -0,0 +1,54 @@ +//===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// RPCUtils implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/RPCUtils.h" + +char llvm::orc::rpc::RPCFatalError::ID = 0; +char llvm::orc::rpc::ConnectionClosed::ID = 0; +char llvm::orc::rpc::ResponseAbandoned::ID = 0; +char llvm::orc::rpc::CouldNotNegotiate::ID = 0; + +namespace llvm { +namespace orc { +namespace rpc { + +std::error_code ConnectionClosed::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCConnectionClosed); +} + +void ConnectionClosed::log(raw_ostream &OS) const { + OS << "RPC connection already closed"; +} + +std::error_code ResponseAbandoned::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCResponseAbandoned); +} + +void ResponseAbandoned::log(raw_ostream &OS) const { + OS << "RPC response abandoned"; +} + +CouldNotNegotiate::CouldNotNegotiate(std::string Signature) + : Signature(std::move(Signature)) {} + +std::error_code CouldNotNegotiate::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCCouldNotNegotiateFunction); +} + +void CouldNotNegotiate::log(raw_ostream &OS) const { + OS << "Could not negotiate RPC function " << Signature; +} + + +} // end namespace rpc +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp new file mode 100644 index 000000000000..b22ecd5f80a1 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -0,0 +1,220 @@ +//===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" + +namespace { + +using namespace llvm; +using namespace llvm::orc; + +class JITDylibSearchOrderResolver : public JITSymbolResolver { +public: + JITDylibSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} + + void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + SymbolNameSet InternedSymbols; + + // Intern the requested symbols: lookup takes interned strings. + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // Build an OnResolve callback to unwrap the interned strings and pass them + // to the OnResolved callback. + // FIXME: Switch to move capture of OnResolved once we have c++14. + auto OnResolvedWithUnwrap = + [OnResolved](Expected<SymbolMap> InternedResult) { + if (!InternedResult) { + OnResolved(InternedResult.takeError()); + return; + } + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + OnResolved(Result); + }; + + // Register dependencies for all symbols contained in this set. + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + MR.addDependenciesForAll(Deps); + }; + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, + OnResolvedWithUnwrap, RegisterDependencies); + } + + Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { + LookupSet Result; + + for (auto &KV : MR.getSymbols()) { + if (Symbols.count(*KV.first)) + Result.insert(*KV.first); + } + + return Result; + } + +private: + MaterializationResponsibility &MR; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} + +void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + + // This method launches an asynchronous link step that will fulfill our + // materialization responsibility. We need to switch R to be heap + // allocated before that happens so it can live as long as the asynchronous + // link needs it to (i.e. it must be able to outlive this method). + auto SharedR = std::make_shared<MaterializationResponsibility>(std::move(R)); + + auto &ES = getExecutionSession(); + + // Create a MemoryBufferRef backed MemoryBuffer (i.e. shallow) copy of the + // the underlying buffer to pass into RuntimeDyld. This allows us to hold + // ownership of the real underlying buffer and return it to the user once + // the object has been emitted. + auto ObjBuffer = MemoryBuffer::getMemBuffer(O->getMemBufferRef(), false); + + auto Obj = object::ObjectFile::createObjectFile(*ObjBuffer); + + if (!Obj) { + getExecutionSession().reportError(Obj.takeError()); + SharedR->failMaterialization(); + return; + } + + // Collect the internal symbols from the object file: We will need to + // filter these later. + auto InternalSymbols = std::make_shared<std::set<StringRef>>(); + { + for (auto &Sym : (*Obj)->symbols()) { + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) { + if (auto SymName = Sym.getName()) + InternalSymbols->insert(*SymName); + else { + ES.reportError(SymName.takeError()); + R.failMaterialization(); + return; + } + } + } + } + + auto K = R.getVModuleKey(); + RuntimeDyld::MemoryManager *MemMgr = nullptr; + + // Create a record a memory manager for this object. + { + auto Tmp = GetMemoryManager(); + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + MemMgrs.push_back(std::move(Tmp)); + MemMgr = MemMgrs.back().get(); + } + + JITDylibSearchOrderResolver Resolver(*SharedR); + + // FIXME: Switch to move-capture for the 'O' buffer once we have c++14. + MemoryBuffer *UnownedObjBuffer = O.release(); + jitLinkForORC( + **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, + [this, K, SharedR, &Obj, InternalSymbols]( + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) { + return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), + ResolvedSymbols, *InternalSymbols); + }, + [this, K, SharedR, UnownedObjBuffer](Error Err) { + std::unique_ptr<MemoryBuffer> ObjBuffer(UnownedObjBuffer); + onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err)); + }); +} + +Error RTDyldObjectLinkingLayer::onObjLoad( + VModuleKey K, MaterializationResponsibility &R, object::ObjectFile &Obj, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> Resolved, + std::set<StringRef> &InternalSymbols) { + SymbolFlagsMap ExtraSymbolsToClaim; + SymbolMap Symbols; + for (auto &KV : Resolved) { + // Scan the symbols and add them to the Symbols map for resolution. + + // We never claim internal symbols. + if (InternalSymbols.count(KV.first)) + continue; + + auto InternedName = getExecutionSession().intern(KV.first); + auto Flags = KV.second.getFlags(); + + // Override object flags and claim responsibility for symbols if + // requested. + if (OverrideObjectFlags || AutoClaimObjectSymbols) { + auto I = R.getSymbols().find(InternedName); + + if (OverrideObjectFlags && I != R.getSymbols().end()) + Flags = I->second; + else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) + ExtraSymbolsToClaim[InternedName] = Flags; + } + + Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) + return Err; + + R.notifyResolved(Symbols); + + if (NotifyLoaded) + NotifyLoaded(K, Obj, *LoadedObjInfo); + + return Error::success(); +} + +void RTDyldObjectLinkingLayer::onObjEmit( + VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer, + MaterializationResponsibility &R, Error Err) { + if (Err) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + R.notifyEmitted(); + + if (NotifyEmitted) + NotifyEmitted(K, std::move(ObjBuffer)); +} + +LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer( + ExecutionSession &ES, ResourcesGetter GetResources, + NotifyLoadedFtor NotifyLoaded, NotifyFinalizedFtor NotifyFinalized, + NotifyFreedFtor NotifyFreed) + : ES(ES), GetResources(std::move(GetResources)), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), + NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp new file mode 100644 index 000000000000..4cb7376758a7 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -0,0 +1,64 @@ +//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities +//h-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Transforms/Utils/Cloning.h" + +namespace llvm { +namespace orc { + +ThreadSafeModule cloneToNewContext(ThreadSafeModule &TSM, + GVPredicate ShouldCloneDef, + GVModifier UpdateClonedDefSource) { + assert(TSM && "Can not clone null module"); + + if (!ShouldCloneDef) + ShouldCloneDef = [](const GlobalValue &) { return true; }; + + auto Lock = TSM.getContextLock(); + + SmallVector<char, 1> ClonedModuleBuffer; + + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(*TSM.getModule(), VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDef(*GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); + + if (UpdateClonedDefSource) + for (auto *GV : ClonedDefsInSrc) + UpdateClonedDefSource(*GV); + + BitcodeWriter BCWriter(ClonedModuleBuffer); + + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } + + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + ThreadSafeContext NewTSCtx(llvm::make_unique<LLVMContext>()); + + auto ClonedModule = + cantFail(parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext())); + ClonedModule->setModuleIdentifier(TSM.getModule()->getName()); + return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx)); +} + +} // end namespace orc +} // end namespace llvm |